Here is a sample of release code for ikea web scraper.
This code developed using our internal library for c# but you can adjust it for your case without using our internal dll.
Just replace data loading function with standart .net webclient or web request.
using System; using System.Collections.Generic; using System.Text; using System.Text.RegularExpressions; using System.Globalization; using System.Web; using CatalogLoaderCommon; using HtmlAgilityPack; using System.Linq; using HtmlDocument = HtmlAgilityPack.HtmlDocument; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using System.Net; using RestSharp.Extensions; #if NET462 using Microsoft.CSharp; // for dynamics #endif namespace CatalogLoader._MyWebScrapers { public class ikea_scraper : CustomScriptBase { public HtmlPageLoader HplCatalog { get; set; } private Product _product; private Category _category; private TaskInfo _mti; public ikea_scraper() { } internal enum Region { EN, PL, RU } internal sealed class IkeaSettings { public RegionSettings RegionSettings { get; set; } public Region Region { get; set; } public List<string> StoreIDs { get; set; } public int ProductQtyMin { get; set; } public double ProductPriceMin { get; set; } public bool CatalogModeEnabled { get; set; } public bool PackagesWeightTotalAsAttribute { get; set; } public bool ProductSizesAsPackagesSizesTotal { get; set; } } internal sealed class RegionSettings { public string Domain { get; set; } public string RootCategoryUrl { get; set; } public string ShopAllDetectorString { get; set; } public string ProductAvailabilityUrlTemplate { get; set; } public string MoreProductsUrl { get; set; } } public override void GrabCatalogBuild(GrabCatalogBuildScriptParameters p) { //Init(p); _category = new Category() { ID = "0", SourceUrl = _ikeaSettings.RegionSettings.RootCategoryUrl }; var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty(); hpl.Load(_category.SourceUrl); CatalogBuild(hpl.HtmlDoc.DocumentNode, _category); foreach (var bottom in _category.CategoriesBottomGet()) { CatalogBuildBottoms(bottom); } //foreach (var c in _category.GetCategoryList()) //{ // c.SourceUrl = CategoryUrlWithIdCompose(c); //} p.Root = _category; } private void CatalogBuildBottoms(Category parent) { if (!_mti.CanContinue()) { return; } _mti.AddLogFormatted("!!! CatalogBuildBottoms() started. Parent => {0}.", parent); var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty(); if (hpl.Load(parent.SourceUrl)) { var subLinks = hpl.HtmlDoc.DocumentNode.SelectNodes(".//div[@class='plp-page-container__main']//div[@class='plp-navigation-slot-wrapper']//nav[@role='navigation'][@data-visual-navigation-links]/a"); if (subLinks != null) { foreach (var link in subLinks) { string name = link.InnerText; string href = link.GetAttributeValue("href", ""); string data_tracking_label = link.GetAttributeValue("data-tracking-label", ""); if (string.IsNullOrEmpty(name) || string.IsNullOrEmpty(href) || AlreadyStoredForParent(parent, href)) { continue; } var sub = new Category() { Name = HttpUtility.HtmlDecode(name.Trim()), SourceUrl = href }; string id = CategoryIdParse(data_tracking_label); if (!string.IsNullOrEmpty(id)) { sub.ID = id; } parent.AddCategory(sub); CatalogBuildBottoms(sub); } } } } private bool AlreadyStoredForParent(Category parent, string possibleNewHref) { return parent.GetParents(true).Any(c => c.SourceUrl == possibleNewHref) || parent.GetCategoryList(true).Any(c => c.SourceUrl == possibleNewHref); } private void CatalogBuild(HtmlNode node, Category parent) { if (node == null) { return; } var subNodes = node.SelectNodes("./li"); if (subNodes != null) { foreach (var subNode in subNodes) { var link = subNode.SelectSingleNode("./a"); if (link != null) { var href = link.GetAttributeValue("href", ""); var data_tracking_label = link.GetAttributeValue("data-tracking-label", ""); if (string.IsNullOrEmpty(href) || !string.IsNullOrEmpty(data_tracking_label) && data_tracking_label.Equals("all", StringComparison.InvariantCultureIgnoreCase) || link.InnerText.IndexOf(_ikeaSettings.RegionSettings.ShopAllDetectorString, StringComparison.InvariantCultureIgnoreCase) >= 0) { continue; } var ul = subNode.SelectSingleNode("./nav/ul"); var sub = new Category() { Name = HttpUtility.HtmlDecode(link.InnerText.Trim()), SourceUrl = href }; if (!string.IsNullOrEmpty(data_tracking_label)) { sub.ID = data_tracking_label; } else { if (ul != null) { var ul_data_tracking_label = ul.GetAttributeValue("data-tracking-label", ""); string catId = CategoryIdParse(ul_data_tracking_label); if (!string.IsNullOrEmpty(catId)) { sub.ID = catId; } } } parent.AddCategory(sub); CatalogBuild(ul, sub); } } } } private string CategoryUrlWithIdCompose(Category cat, string id = null) { if (string.IsNullOrEmpty(id)) { id = cat.ID; } return string.Format("{0}@@{1}", cat.SourceUrl, id); } private string CategoryIdParse(string data_tracking_label) { if (!string.IsNullOrEmpty(data_tracking_label)) { var arr = data_tracking_label.Split(new[] { "|" }, StringSplitOptions.RemoveEmptyEntries); if (arr.Length > 0) { return arr[arr.Length - 1].Trim(); } } return ""; } private GrabProcessState _gps; private CacheCustom _cacheCustom = null; private IkeaSettings _ikeaSettings; public override void Login(LoginScriptParameters p) { _gps = p.State as GrabProcessState; _mti = p.Process.m_ti; p.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36"; _ikeaSettings = IkeaSettingsBuild(_gps.GrabberSettings); _ikeaSettings.RegionSettings = RegionSettingsBuild(_ikeaSettings.Region); _gps.GrabberSettings.Settings.CacheEnabled = true; // Make enabled by default. _cacheCustom = new CacheCustom(_gps.CacheGet().m_cacheStorage); } private IkeaSettings IkeaSettingsBuild(GrabberSettings gs) { var ikeaSettings = new IkeaSettings(); string minQty = gs.UserParameterGet("ProductQtyMin"); if (!string.IsNullOrEmpty(minQty)) { int mq = 0; if (int.TryParse(minQty, out mq)) { ikeaSettings.ProductQtyMin = mq; } } string minPrice = gs.UserParameterGet("ProductPriceMin"); if (!string.IsNullOrEmpty(minPrice)) { double mp = 0; if (double.TryParse(minPrice, out mp)) { ikeaSettings.ProductPriceMin = mp; } } string regionName = gs.UserParameterGet("Region"); if (string.IsNullOrEmpty(regionName)) { regionName = "EN"; } Region r; if (Enum.TryParse(regionName.ToUpper(), out r)) { ikeaSettings.Region = r; } ikeaSettings.CatalogModeEnabled = true; string catalogModeEnabled = gs.UserParameterGet("CatalogModeEnabled"); if (!string.IsNullOrEmpty(catalogModeEnabled)) { bool b; if (bool.TryParse(catalogModeEnabled, out b)) { ikeaSettings.CatalogModeEnabled = b; } } string pckgsWeightTotalAsAttr = gs.UserParameterGet("PackagesWeightTotalAsAttribute"); if (!string.IsNullOrEmpty(pckgsWeightTotalAsAttr)) { bool b; if (bool.TryParse(pckgsWeightTotalAsAttr, out b)) { ikeaSettings.PackagesWeightTotalAsAttribute = b; } } string prodSizesAsPckgsSizesTotal = gs.UserParameterGet("ProductSizesAsPackagesSizesTotal"); if (!string.IsNullOrEmpty(prodSizesAsPckgsSizesTotal)) { bool b; if (bool.TryParse(prodSizesAsPckgsSizesTotal, out b)) { ikeaSettings.ProductSizesAsPackagesSizesTotal = b; } } string storeIds = gs.UserParameterGet("StoreId"); if (!string.IsNullOrEmpty(storeIds)) { ikeaSettings.StoreIDs = storeIds.Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries).ToList(); } return ikeaSettings; } private RegionSettings RegionSettingsBuild(Region r) { var rs = new RegionSettings(); switch (r) { case Region.EN: rs.Domain = "https://www.ikea.com/us/en/"; rs.RootCategoryUrl = "https://www.ikea.com/us/en/header-footer/menu-products.html"; rs.ShopAllDetectorString = "Shop all"; rs.ProductAvailabilityUrlTemplate = "https://api.ingka.ikea.com/cia/availabilities/ru/us?itemNos={0}&expand=StoresList,Restocks"; rs.MoreProductsUrl = "https://sik.search.blue.cdtapps.com/us/en/product-list-page/more-products?category=&sort=RELEVANCE&start=&end=&c=lf&v=20200617"; break; case Region.PL: rs.Domain = "https://www.ikea.com/pl/pl/"; rs.RootCategoryUrl = "https://www.ikea.com/pl/pl/header-footer/menu-products.html"; rs.ShopAllDetectorString = "Zobacz wszystko"; rs.ProductAvailabilityUrlTemplate = "https://api.ingka.ikea.com/cia/availabilities/ru/pl?itemNos={0}&expand=StoresList,Restocks"; rs.MoreProductsUrl = "https://sik.search.blue.cdtapps.com/pl/pl/product-list-page/more-products?category=&sort=RELEVANCE&start=&end=&c=lf&v=20200617"; break; case Region.RU: rs.Domain = "https://www.ikea.com/ru/ru/"; rs.RootCategoryUrl = "https://www.ikea.com/ru/ru/header-footer/menu-products.html"; rs.ShopAllDetectorString = "Смотреть все"; rs.ProductAvailabilityUrlTemplate = "https://api.ingka.ikea.com/cia/availabilities/ru/ru?itemNos={0}&expand=StoresList,Restocks"; rs.MoreProductsUrl = "https://sik.search.blue.cdtapps.com/ru/ru/product-list-page/more-products?category=&sort=RELEVANCE&start=&end=&c=lf&v=20200617"; break; default: break; } return rs; } private Dictionary<string, Dictionary<string, string>> _mainProductUrl2VariantsIds2Links = new Dictionary<string, Dictionary<string, string>>(); public override void GetProductLinksForCategory(CatalogLoaderCommon.GetProductLinksForCategoryScriptParameters p) { var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty(); if (!hpl.Load(p.Category.SourceUrl)) { return; } _mti.AddLogInfo($"_ikeaSettings.CatalogModeEnabled ? {_ikeaSettings.CatalogModeEnabled}"); if (_ikeaSettings.CatalogModeEnabled) { Paginator paginator = new Paginator(_ikeaSettings.RegionSettings); string msg; if (!paginator.Init(hpl.HtmlDoc, out msg)) { _mti.AddLogError(string.Format("Paginator init failed. Message:'{0}'.", msg)); return; } int newLinksCnt = 0; while (paginator.MoveNext()) { string nextPage = paginator.MoreProductsLinkBuild(); List<string> links = new List<string>(); if (hpl.Load(nextPage)) { try { var resp = JsonConvert.DeserializeObject<ModelsIkea.Pagination.MoreproductsResponse>(hpl.Content); //links = resp.moreProducts.productWindow.Select(pw => pw.pipUrl); foreach (var pw in resp.moreProducts.productWindow) { links.Add(pw.pipUrl); /* * https://www.ikea.com/ru/ru/cat/vazy-dlya-cvetov-10776/ * http://storage.mydataprovider.com/screenshots/akl/2021-02-11_14-47-22_FLmTFPxU.png * * Не у всех товаров на их странице есть варианты, * хотя на странице категории варианты у товара могут быть. * В MoreproductsResponse теперь приходит информация о вариантах. * Поэтому сохраняем варианты для ссылок на основной товар, чтобы взять * их при отсутствии вариантов на странице товара. * */ if (pw.gprDescription != null && pw.gprDescription.variants != null) { _mainProductUrl2VariantsIds2Links[pw.pipUrl] = pw.gprDescription.variants.ToDictionary(v => v.id, v => v.pipUrl); } } } catch (Exception ex) { _mti.AddLogError(ex.ToString()); } } else { break; } if (links == null || !links.Any()) { _mti.AddLogInfo("Can't find new links."); break; } foreach (var link in links) { p.Category.ProductLinks.Add(link); newLinksCnt++; if (newLinksCnt == hpl.Settings.MaxProductsInCategory) { goto OUT_OF_PAGIN; } } } OUT_OF_PAGIN: _mti.AddLogFormatted("GetProductLinksForCategory finished. Links count:{0}.", newLinksCnt.ToString()); } else { _gps.CacheGet().OnPageLoadedWithError += Cache_OnPageLoadedWithError; // В этих настройках парсим ссылки на все товары из категорий. // https://www.ikea.com/ru/ru/header-footer/menu-products.html // http://storage.mydataprovider.com/screenshots/akl/2020-10-23_09-11-43_XHHft6PA.png // Если товаров в ней нет, идем в подкатегории. // ================================================================================== // // akl_03122020 // !!! UPDATE: // // Не во всех категориях "Смотреть все товары" возвращаются реально все товары. // Напр., "Товары для ремонта" https://www.ikea.com/ru/ru/cat/tovary-dlya-remonta-hi001/ // не содержит все товары // из "Шурупы и крепления" https://www.ikea.com/ru/ru/cat/shurupy-i-krepleniya-49768/ // Поэтому, начинаем обход с подкатегорий. try { TestCategory testCategory = null; /* testCategory = new TestCategory() { Root = "Кухни и бытовая техника", Sub = "Кухни", BottomId = "ka003" }; */ if (!string.IsNullOrEmpty(_gps.GrabberSettings.UserParameterGet("TEST_CATEGORY"))) { try { testCategory = JsonConvert.DeserializeObject<TestCategory>(_gps.GrabberSettings.UserParameterGet("TEST_CATEGORY")); } catch { } } AllProductsProcess(p, testCategory: testCategory); } catch (Exception ex) { _mti.AddLogError(ex); } finally { _gps.CacheGet().OnPageLoadedWithError -= Cache_OnPageLoadedWithError; //_productsPrcsdLinks.Clear(); } _mti.AddLogFormatted("GetProductLinksForCategory finished. Links count:{0}.", _productsPrcsdLinks.Count.ToString()); } } public class TestCategory { private string _rootXpath; private string _subXpath; public string Root { get; set; } public string Sub { get; set; } internal string RootXpath { get { if (string.IsNullOrEmpty(_rootXpath)) { _rootXpath = string.Format("./li/a[contains(.,'{0}')]", Root); } return _rootXpath; } } internal string SubXpath { get { if (string.IsNullOrEmpty(_subXpath)) { _subXpath = string.Format("./nav/ul/li/a[contains(.,'{0}')]", Sub); } return _subXpath; } } public string BottomId { get; set; } } private List<string> _productsPrcsdLinks = new List<string>(100000); private Dictionary<string, string> _categoryName2Url = new Dictionary<string, string>(); private Dictionary<string, string> _productUrl2FullCategoryName = new Dictionary<string, string>(); private void AllProductsProcess(GetProductLinksForCategoryScriptParameters p, List<string> allProductsLinks = null, TestCategory testCategory = null) { int maxProductsInCategory = 100000; var allProductsLink2AllProductsLinkNode = new Dictionary<string, HtmlNode>(); var allProductsLink2CategoryName = new Dictionary<string, string>(); var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty(); if (allProductsLinks == null) { allProductsLinks = new List<string>(); hpl.Load(p.Category.SourceUrl); string categoriesLinksXPath = testCategory != null ? testCategory.RootXpath : "./li/a"; var categoriesLinks = hpl.HtmlDoc.DocumentNode.SelectNodes(categoriesLinksXPath); if (categoriesLinks == null) { _mti.AddLogWarning("!!! Can't parse categories links."); return; } foreach (var catLink in categoriesLinks) { if (!_mti.CanContinue() || string.IsNullOrEmpty(catLink.InnerText)) { continue; } var categoryName = catLink.InnerText.Trim(); _categoryName2Url[categoryName] = TextUtils.UrlNewCompose(_ikeaSettings.RegionSettings.Domain, catLink.Attributes["href"].Value); string subCategoriesLinksXPath = testCategory != null ? testCategory.SubXpath : $"./nav/ul/li/a[not(contains(.,'{_ikeaSettings.RegionSettings.ShopAllDetectorString}'))]"; var subCategoriesLinksNodes = catLink.ParentNode.SelectNodes(subCategoriesLinksXPath); if (subCategoriesLinksNodes == null) { _mti.AddLogWarning(string.Format("!!! Category:'{0}' => can't find subcategories link.", categoryName)); continue; } _mti.AddLogWarning(string.Format("!!! Category:'{0}' => subcategories links count:{1}.", categoryName, subCategoriesLinksNodes.Count.ToString())); foreach (var subCategoryNode in subCategoriesLinksNodes) { if (!_mti.CanContinue()) { continue; } string subCategoryLink = subCategoryNode.GetAttributeValue("href", ""); if (!string.IsNullOrEmpty(subCategoryLink)) { string subCategoryName = subCategoryNode.InnerText.Trim(); subCategoryLink = TextUtils.UrlNewCompose(_ikeaSettings.RegionSettings.Domain, subCategoryLink); _categoryName2Url[subCategoryName] = subCategoryLink; string categoryId = CategoryIdParseFromUrl(subCategoryLink); if (testCategory != null) { categoryId = testCategory.BottomId; } _mti.AddLogInfo(string.Format("!!! Subcategory link:'{0}' => category ID:'{1}'.", subCategoryLink, categoryId)); if (string.IsNullOrEmpty(categoryId)) { continue; } /* * akl_03022021: * Т.к. утоваров без крошек в карточке отсутствует ссылка на их непосредственную категорию, * придется сделать минимум еще один запрос, чтобы собрать имена-ссылки подкатегорий здесь. */ if (hpl.Load(subCategoryLink)) { foreach (var name2url in CategoriesName2UrlParse(hpl.Content)) { _categoryName2Url[name2url.Key] = name2url.Value; } } var links = new List<string>(); Models.Pagination.MoreproductsResponse resp = null; if (hpl.Load(string.Format("https://sik.search.blue.cdtapps.com/ru/ru/product-list-page/more-products?category={0}&sort=RELEVANCE&start=0&end={1}&c=lf&v=20210126", categoryId, maxProductsInCategory.ToString()))) { try { resp = JsonConvert.DeserializeObject<Models.Pagination.MoreproductsResponse>(hpl.Content); //links = resp.moreProducts.productWindow.Select(pw => pw.pipUrl); foreach (var pw in resp.moreProducts.productWindow) { links.Add(pw.pipUrl); /* * https://www.ikea.com/ru/ru/cat/vazy-dlya-cvetov-10776/ * http://storage.mydataprovider.com/screenshots/akl/2021-02-11_14-47-22_FLmTFPxU.png * * Не у всех товаров на их странице есть варианты, * хотя на странице категории варианты у товара могут быть. * В MoreproductsResponse теперь приходит информация о вариантах. * Поэтому сохраняем варианты для ссылок на основной товар, чтобы взять * их при отсутствии вариантов на странице товара. * */ if (pw.gprDescription != null && pw.gprDescription.variants != null) { _mainProductUrl2VariantsIds2Links[pw.pipUrl] = pw.gprDescription.variants.ToDictionary(v => v.id, v => v.pipUrl); } /* * akl_02262021: * Есть товары, в карточке которых отсутствуют крошки. По этой причине, * нужно мапить ссылку товара на полную категорию на этапе сбора ссылок. */ _productUrl2FullCategoryName[pw.pipUrl] = string.Format("{0}->{1}", categoryName, subCategoryName); } } catch (Exception ex) { _mti.AddLogError(ex.ToString()); } } else { continue; } _mti.AddLogFormatted("!!! Category Link '{0}' => products links num by api: {1}.", subCategoryLink, links != null ? links.Count().ToString() : "0"); //if (links == null) //{ // continue; //} foreach (var link in links) { if (_productsPrcsdLinks.Contains(link)) { _mti.AddLogFormatted("!!! Product link '{0}' stored allready.", link); continue; } _productsPrcsdLinks.Add(link); p.Category.ProductLinks.Add(link); if (_productsPrcsdLinks.Count == hpl.Settings.MaxProductsInCategory) { goto OUT_OFF_LOOPS; } } } } } } OUT_OFF_LOOPS: _mti.AddLogInfo("[AllProductsProcess] Finished."); } private string CategoryIdParseFromUrl(string link) { var urlNormal = TextUtils.UrlNewCompose(_ikeaSettings.RegionSettings.Domain, link); Uri u; if (Uri.TryCreate(urlNormal, UriKind.RelativeOrAbsolute, out u)) { return Regex.Replace(u.AbsolutePath, @"^[\w\W]+-", "").TrimEnd('/'); } return ""; } private List<string> _bottomsCategoriesXpathList = new List<string>() { ".//div[contains(@class,'plp-category-link-list')]//li[contains(@class,'plp-category-link-list__item')]/a", ".//nav[@role='navigation']/a" }; private Dictionary<string, string> CategoriesName2UrlParse(string html) { var hDoc = new HtmlDocument(); hDoc.LoadHtml(html); foreach (var xpath in _bottomsCategoriesXpathList) { var nodes = hDoc.DocumentNode.SelectNodes(xpath); if (nodes != null) { var n2u = new Dictionary<string, string>(); foreach (var n in nodes) { string h = n.GetAttributeValue("href", ""); if (string.IsNullOrEmpty(n.InnerText) || string.IsNullOrEmpty(h)) { continue; } n2u[n.InnerText.Trim()] = TextUtils.UrlNewCompose(_ikeaSettings.RegionSettings.Domain, h); } _mti.AddLogInfo("[CategoriesName2UrlParse] Result =================="); _mti.AddLogInfo(string.Join("\r\n", n2u.Select(kvp => string.Format("{0} => {1}", kvp.Key, kvp.Value)))); _mti.AddLogInfo("===================================================="); return n2u; } } return new Dictionary<string, string>(); } private bool Cache_OnPageLoadedWithError(Cache cacheProvider, HttpWebResponse response, out string content) { content = ""; if (response.ContentLength != 0) { _mti.AddLogWarning("In CustomScript_OnPageLoadedWithError"); try { var dataContent = response.GetResponseStream().ReadAsBytes(); var exceptionContent = cacheProvider.Encoding.GetString(dataContent); if (exceptionContent.Contains("Unknown category key")) { content = exceptionContent; return true; } } catch (Exception) { } } return !string.IsNullOrEmpty(content); } internal class Paginator { /* * https://sik.search.blue.cdtapps.com/ru/ru/product-list-page/more-products?sessionId=5980bee3-2658-4788-bb3c-501cd50287f3&category=11703&sort=RELEVANCE&start=24&end=48&c=lf&v=20200617 * Значение параметра запроса "sessionId" совпадает с local storage ключем "search-sik-session-ruru". * Источник возвращает валидные данные и без "sessionId", поэтому, пока что, игнорим этот параметр. */ private int _currentIdx; private int _lastIdx = 0; private int _shift; private ModelsIkea.Pagination.DataCategory _dataCategory; private RegionSettings _regionSettings; public Paginator(RegionSettings rs, int startIdx = 0, int shift = 24) { _regionSettings = rs; _currentIdx = startIdx - 1; _shift = shift; } internal bool Init(HtmlDocument htmlDoc, out string msg) { msg = string.Empty; bool r = false; string dataCategoryStr = TextUtils.GetHtmlValue(htmlDoc, ".//div[contains(@class,'js-product-list')]", "data-category", true); if (!string.IsNullOrEmpty(dataCategoryStr)) { try { _dataCategory = JsonConvert.DeserializeObject<ModelsIkea.Pagination.DataCategory>(dataCategoryStr); _lastIdx = _dataCategory.totalCount / _shift; if (_dataCategory.totalCount % _shift > 0) { _lastIdx++; } r = true; } catch (Exception ex) { msg = ex.ToString(); } } else { msg = "dataCategoryStr is EMPTY"; } return r; } internal string MoreProductsLinkBuild() { var uBuilder = new UriBuilder(_regionSettings.MoreProductsUrl); var query = HttpUtility.ParseQueryString(uBuilder.Query); query["category"] = _dataCategory.id; query["start"] = (_currentIdx * _shift).ToString(); query["end"] = (_currentIdx * _shift + _shift).ToString(); uBuilder.Query = query.ToString(); return uBuilder.ToString(); } internal bool MoveNext() { _currentIdx++; return _currentIdx <= _lastIdx; } } public override void RunCategory(RunCategoryScriptParameters p) { } private void Init(RunProductScriptParameters p) { _mti = p.Process.m_ti; _product = p.Product; _category = p.Category; } public override void RunProduct(RunProductScriptParameters p) { try { Init(p); _category = p.Category; _product.Name = HttpUtility.HtmlDecode(_product.Name); _product.Name = _product.GetAttributeValue("SYS_PRODUCT_NAME2") + " " + _product.Name; if (_cacheCustom.IsProcessed(_product.Art)) { _mti.AddLogFormatted("!!! Product ART:'{0}' will be skipped because processed early.", _product.Art); return; } if (_product.PriceIsOk) _product.Price = _product.Price/*.Replace(".", ",")*/.Replace(" ", string.Empty); else _product.Price = "0"; bool mainVariantProcessed = false; bool variantsHtmlFromProductPage = false; if (!string.IsNullOrEmpty(_product.GetAttributeValue("VARIATION_SECTION"))) { variantsHtmlFromProductPage = true; } else if (!string.IsNullOrEmpty(_product.GetAttributeValue("PRODUCT_STYLES"))) { //https://www.ikea.com/ru/ru/p/kolbyorn-shkaf-d-doma-sada-zelenyy-40455908/ //для товаров с комбинациями только по одной опции. variantsHtmlFromProductPage = true; } Dictionary<string, string> variantsIDs2Links = new Dictionary<string, string>(); if (variantsHtmlFromProductPage) { //VariantsIDs2LinksPopulate(product.GetAttributeValue("VARIATION_SECTION"), product.GetAttributeValue("PRODUCT_STYLES"), // variantsIDs2Links, new List<string>() { product.Url }); variantsIDs2Links = VariantsIDs2LinksGet(_product.GetAttributeValue("VARIATION_SECTION"), _product.GetAttributeValue("PRODUCT_STYLES")); } else { var variantsIDs2LinksStoredFromCategoryPage = new Dictionary<string, string>(); if (_mainProductUrl2VariantsIds2Links.TryGetValue(_product.Url, out variantsIDs2LinksStoredFromCategoryPage)) { _mti.AddLogFormatted("!!! Variants from category page will be used => product url:'{0}'", _product.Url); variantsIDs2Links = variantsIDs2LinksStoredFromCategoryPage; //Нужно принудительно обработать вариант основного товара, если берем варианты из категории. variantsIDs2Links[_product.Art] = _product.Url; } } var nonProcessedVariantsIds2Links = new Dictionary<string, string>(); foreach (var id2v in variantsIDs2Links) { if (_cacheCustom.IsProcessed(id2v.Key)) { _mti.AddLogFormatted("!!! Variant ART:'{0}' will be skipped because processed early.", id2v.Key); } else { nonProcessedVariantsIds2Links[id2v.Key] = id2v.Value; } } if (nonProcessedVariantsIds2Links.Count > 0) { var vIds2LinksForBulkLoading = new Dictionary<string, string>(); foreach (var id2Link in nonProcessedVariantsIds2Links) { string html; if (!_cacheCustom.Load(id2Link.Value, out html)) { vIds2LinksForBulkLoading[id2Link.Key] = id2Link.Value; } } if (vIds2LinksForBulkLoading.Count > 0) { VariantsBulkLoad(vIds2LinksForBulkLoading); } var cmbs = new List<Combination>(); foreach (var varId2Link in nonProcessedVariantsIds2Links) { Product productForProcessing; if (varId2Link.Key.Equals(_product.Art, StringComparison.InvariantCultureIgnoreCase)) { productForProcessing = _product; mainVariantProcessed = true; } else { productForProcessing = new Combination(); productForProcessing.Art = productForProcessing.ID = varId2Link.Key.ToUpper(); productForProcessing.Url = varId2Link.Value; } if (VariantProcessing(productForProcessing)) { var cmb = productForProcessing as Combination; if (cmb != null) { cmbs.Add(cmb); } } } _product.CombinationsAdd(cmbs); _product.SetAttributeValue("VARIATION_SECTION", ""); /* * Нужно обрабытывать основной товар даже при наличии у него вариантов, т.к. некоторые товары не попадают в свои варианты. * Напр. https://www.ikea.com/ru/ru/p/metod-navesnoy-shkaf-so-steklyannoy-dveryu-belyy-budbin-seryy-s89227745/ */ if (!mainVariantProcessed) { VariantProcessing(_product); mainVariantProcessed = true; } var productsForSave = new List<Product>() { _product }; foreach (var prod in ProductsFromCombinationsGet(_product)) { if (productsForSave.Any(pr => pr.Art == prod.Art)) { continue; } productsForSave.Add(prod); } _product.CombinationsRemove(); var grabHelper = _gps.ProcessMain as GrabHelper; foreach (var pr in productsForSave) { if (_cacheCustom.IsProcessed(pr.Art)) { _mti.AddLogFormatted("!!! Product ART:'{0}' will be skipped because processed early.", pr.Art); continue; } _cacheCustom.SaveAsProcessed(pr.Art); if ((_ikeaSettings.ProductQtyMin > 0 && pr.QuantityInt() < _ikeaSettings.ProductQtyMin) || (_ikeaSettings.ProductPriceMin > 0 && pr.PriceDoubleUnsafe < _ikeaSettings.ProductPriceMin)) { continue; } var tempProd = HtmlBlocksClear(pr); tempProd.Category = p.Category; ProductImagesRename(tempProd); ProductImagesNumberTruncate(tempProd, 40); tempProd.Art = tempProd.ID = IDCreate(tempProd.Art); string strError = null; if (grabHelper == null || !grabHelper._grabProcessParameters.EmbProvider.ProductSave(tempProd, out strError)) { _mti.AddLogError(string.Format("Database provider: {0}", strError)); } } } ////if (_productQtyMin > 0 && product.QuantityInt() < _productQtyMin) //{ //product.SavedToStorage = product.SaveIt = false; //} } finally { _product.SavedToStorage = _product.SaveIt = false; _cacheCustom.HtmlCacheClear(); } } private void VariantsBulkLoad(Dictionary<string, string> variantsIDs2Links) { _mti.AddLogFormatted("[VariantsBulkLoad] Started. variantsIDs2Links:{0}.", variantsIDs2Links.Count); var bulkRequests = new List<CatalogLoader.Utils.HtmlBulkRequests.HttpWebRequestEntry>(); foreach (var id2Link in variantsIDs2Links) { var r = HttpWebRequest.Create(id2Link.Value) as HttpWebRequest; r.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36"; r.CookieContainer = _gps.m_cookies_con; bulkRequests.Add(new CatalogLoader.Utils.HtmlBulkRequests.HttpWebRequestEntry() { Key = id2Link.Key, Request = r }); } var hbr = HtmlBulkRequestsGet(); _mti.AddLogFormatted("[VariantsBulkLoad] !!! hbr.Process() started => hbr._requests.Count:{0}.", bulkRequests.Count.ToString()); hbr.Process(bulkRequests); _mti.AddLogFormatted("[VariantsBulkLoad] !!! hbr.Process() finished => hbr._unprocessedRequests.Count:{0}.", hbr._unprocessedRequests.Count.ToString()); foreach (var id2Link in variantsIDs2Links) { var requestEntry = _hbr._responses.FirstOrDefault(t => t.Key == id2Link.Key); if (requestEntry.Failed) { throw new Exception(string.Format("[VariantsBulkLoad] request entry with key:'{0}' failed.", requestEntry.Key)); } _cacheCustom.SaveHtml(id2Link.Value, requestEntry.Content); } _mti.AddLogInfo("[VariantsBulkLoad] Finished."); } internal sealed class QtyItem { public int Value { get; set; } public string StoreId { get; set; } public override string ToString() { return $"Value:{Value}, StoreId:{StoreId}"; } } internal sealed class CacheCustom { private CacheStorage _cacheStorage; private Dictionary<string, string> _url2Html; private Dictionary<string, List<QtyItem>> _art2Qty; private HashSet<string> _productsArtProcessed = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase); public CacheCustom(CacheStorage cacheStorage) { _cacheStorage = cacheStorage; _url2Html = new Dictionary<string, string>(); _art2Qty = new Dictionary<string, List<QtyItem>>(); } internal bool IsCached(string url) { string html; if (!_url2Html.TryGetValue(url, out html)) { if (_cacheStorage.Load(url, out html)) { _url2Html[url] = html; } } return !string.IsNullOrEmpty(html); } internal bool Load(string url, out string html) { html = ""; if (IsCached(url)) { html = _url2Html[url]; return true; } return false; } internal void SaveHtml(string url, string html) { if (!string.IsNullOrEmpty(url)) { _url2Html[url] = html; _cacheStorage.Save(url, html); } } internal void SaveQty(string url, QtyItem qty) { if (!string.IsNullOrEmpty(url)) { if (!_art2Qty.TryGetValue(url, out List<QtyItem> qts)) { qts = new List<QtyItem>(); _art2Qty[url] = qts; } qts.Add(qty); } } internal void HtmlCacheClear() { _url2Html.Clear(); } internal bool GetQty(string art, out List<QtyItem> qtyLIst) { return _art2Qty.TryGetValue(art, out qtyLIst); } internal bool IsProcessed(string art) { if (!string.IsNullOrEmpty(art)) { return _productsArtProcessed.Contains(art); } return false; } internal void SaveAsProcessed(string art) { if (!string.IsNullOrEmpty(art)) { _productsArtProcessed.Add(art); } } } //private Dictionary<string, int> _variantId2Qty = new Dictionary<string, int>(); CatalogLoader.Utils.HtmlBulkRequests _hbr = null; private Dictionary<string, string> VariantsIDs2LinksGet(string variationSection, string productStyles) { /* * akl_12102020 * * http://crm.catalogloader.com/issues/35427#note-88 * https://www.ikea.com/ru/ru/p/longfell-rabochiy-stul-s-podlokotnikami-gunnared-bezhevyy-chernyy-s29210026/ * "В выгрузку попадают только товары с разным основанием: белый, чёрный. Разные цвета чехлов не попадают - только бежевый. Таким образом из 8 вариантов в xml попали 2." * ========================================================================================================= * * Нужно перебирать все варианты ("VARIATION_SECTION"), переходить в карточку каждого * и там уже собирать стили ("PRODUCT_STYLES"). * Иначе, теряются комбинации стилей. * ========================================================================================================== * * akl_18112020 * * http://crm.catalogloader.com/issues/35427#note-111 * "Предложений должно быть 32, а выгружаеся только 8" * https://www.ikea.com/ru/ru/p/malm-karkas-krovati-dubovyy-shpon-belenyy-luroy-s39210945/ * * !!! UP: После перехода в карточку варианта, нужно собирать не только стили, но и сами варианты. * Иначе, список комбинаций будет неполным. * * akl_06122021 * Похоже, переход по ссылке каждого варианта больше не актуален - берем только то, * что пришло на странице основного товара и на странице каждого стиля. */ _mti.AddLogInfo("[VariantsIDs2LinksGet] Started."); var ids2Variants = new Dictionary<string, Variant>(); // Берем все варианты стилей, если есть... var stylesId2Variant = new Dictionary<string, Variant>(); if (!string.IsNullOrEmpty(productStyles)) { foreach (var id2Variant in Ids2VariantsParse(productStyles, typeof(ModelsIkea.Variations.Variation))) { if (!_mti.CanContinue()) { break; } stylesId2Variant[id2Variant.Key] = id2Variant.Value; } } // ...собираем варианты на странице каждого неактивного стиля... foreach (var styleId2Variant in stylesId2Variant) { if (!_mti.CanContinue()) { break; } if (!styleId2Variant.Value.IsSelected) { var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty(); hpl.Load(styleId2Variant.Value.Url); var variantsNode = hpl.HtmlDoc.DocumentNode.SelectSingleNode(".//div[contains(@class,'js-product-variation-section')]"); if (variantsNode != null) { string VARIATION_SECTION = HttpUtility.HtmlDecode(variantsNode.GetAttributeValue("data-initial-props", "")); if (!string.IsNullOrEmpty(VARIATION_SECTION)) { foreach (var id2Variant in Ids2VariantsParse(VARIATION_SECTION, typeof(ModelsIkea.Variations.VariationsSection))) { if (!_mti.CanContinue()) { break; } ids2Variants[id2Variant.Key] = id2Variant.Value; } } else { //...если вариантов нет, то берем сам стиль... ids2Variants[styleId2Variant.Key] = styleId2Variant.Value; } } } } // ...собираем варианты на странице активного стиля... if (!string.IsNullOrEmpty(variationSection)) { foreach (var id2Variant in Ids2VariantsParse(variationSection, typeof(ModelsIkea.Variations.VariationsSection))) { if (!_mti.CanContinue()) { break; } ids2Variants[id2Variant.Key] = id2Variant.Value; } } else { var styleSelectedId2Var = stylesId2Variant.FirstOrDefault(id2v => id2v.Value.IsSelected); if (!string.IsNullOrEmpty(styleSelectedId2Var.Key)) { ids2Variants[styleSelectedId2Var.Key] = styleSelectedId2Var.Value; } } var nonProcessedVariants = new List<Variant>(); foreach (var id2v in ids2Variants) { if (_cacheCustom.IsProcessed(id2v.Key)) { _mti.AddLogFormatted("!!! Variant ART:'{0}' will be skipped because processed early.", id2v.Key); } else if (_ikeaSettings.ProductPriceMin > 0 && id2v.Value.Price < _ikeaSettings.ProductPriceMin) { _mti.AddLogFormatted("!!! Variant '{0}' processing interrupt -> current price ({1}) < minimum price allowable value ({2})", id2v.Key, id2v.Value.Price.ToString(), _ikeaSettings.ProductPriceMin.ToString()); _cacheCustom.SaveAsProcessed(id2v.Key); } else { nonProcessedVariants.Add(id2v.Value); } } List<Variant> varsValid = VariantsAvailabilityValidation(nonProcessedVariants); _mti.AddLogInfo("[VariantsIDs2LinksGet] Finished."); return varsValid.Count > 0 ? varsValid.ToDictionary(v => v.Id, v => v.Url) : new Dictionary<string, string>(); } CatalogLoader.Utils.HtmlBulkRequests HtmlBulkRequestsGet() { if (_hbr == null) { _hbr = new CatalogLoader.Utils.HtmlBulkRequests(); _hbr.ThreadsNumber = 20; _hbr.ProxySourceList = "http://ru.proxy-doctor.com/WebApi/CampaignDataLatestGet/1-201705051618-NNCIAPGJEYMVQKGXQRXKFKCOKJLDNMRVBFBQQMLXHPQHYBMDXK?collection-name=ikearu"; _hbr.BanDetectionString = "The following error was encountered while trying to retrieve the URL[next]Method Not Allowed[next]Access Denied[next]非"; //_hbr.RequestAttempts = _gps.GrabberSettings.Settings.RequestAttempts; _hbr.RequestAttempts = 10; _hbr.RequestTimeout = _gps.GrabberSettings.Settings.RequestTimeout; _hbr.CustomRequestUse = true; } return _hbr; } private List<Variant> VariantsAvailabilityValidation(List<Variant> variants) { _mti.AddLogFormatted("[VariantsAvailabilityValidation] Started. variants total:{0}.", variants.Count.ToString()); var varsToProcess = new List<Variant>(); var id2IdNorm = new Dictionary<string, string>(); foreach (var v in variants) { if (!_cacheCustom.GetQty(v.Id, out List<QtyItem> qts)) { varsToProcess.Add(v); } } if (varsToProcess.Count > 0) { var bulkRequests = new List<CatalogLoader.Utils.HtmlBulkRequests.HttpWebRequestEntry>(); _mti.AddLogFormatted("[VariantsAvailabilityValidation] !!!varsToProcess.Count:{0}.", varsToProcess.Count.ToString()); var idNorm2Qhtml = new Dictionary<string, string>(varsToProcess.Count); foreach (var vp in varsToProcess) { string idNorm = vp.Id.ToUpper(); if (idNorm.StartsWith("S")) { idNorm = idNorm.TrimStart('S'); } id2IdNorm[vp.Id] = idNorm; //var aviTest = ProductAvailabilityInfoGet(_gps.GrabberSettings.UserParameterGet("StoreId"), id2IdNorm[vp.Id]); //string qUrl = string.Format("https://api.ingka.ikea.com/cia/availabilities/ru/us?itemNos={0}&expand=StoresList,Restocks", idNorm); string qUrl = string.Format(_ikeaSettings.RegionSettings.ProductAvailabilityUrlTemplate, idNorm); string qHtml; if (!_cacheCustom.Load(qUrl, out qHtml)) { var r = HttpWebRequest.Create(qUrl) as HttpWebRequest; r.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36"; r.Accept = "application/json;version=2"; r.Headers["X-Client-Id"] = "b6c117e5-ae61-4ef5-b4cc-e0b1e37f0631"; r.Headers["accept-encoding"] = "gzip, deflate, br"; r.CookieContainer = _gps.m_cookies_con; bulkRequests.Add(new CatalogLoader.Utils.HtmlBulkRequests.HttpWebRequestEntry() { Key = vp.Id, Request = r }); } else { idNorm2Qhtml[idNorm] = qHtml; } } var hbr = HtmlBulkRequestsGet(); _mti.AddLogFormatted("[VariantsAvailabilityValidation] !!! hbr.Process() started => hbr._requests.Count:{0}.", bulkRequests.Count.ToString()); hbr.Process(bulkRequests); _mti.AddLogFormatted("[VariantsAvailabilityValidation] !!! hbr.Process() finished => hbr._unprocessedRequests.Count:{0}.", hbr._unprocessedRequests.Count.ToString()); foreach (var vp in varsToProcess) { string qHtml = ""; var requestEntry = _hbr._responses.FirstOrDefault(t => t.Key == vp.Id); if (requestEntry != null) { if (requestEntry.Failed) { throw new Exception(string.Format("[VariantsAvailabilityValidation] request entry with key:'{0}' failed.", requestEntry.Key)); } else { qHtml = requestEntry.Content; _cacheCustom.SaveHtml(requestEntry.Request.Address.AbsoluteUri, requestEntry.Content); } } else { idNorm2Qhtml.TryGetValue(id2IdNorm[vp.Id], out qHtml); } foreach (var storeId in _ikeaSettings.StoreIDs) { var avi = ProductAvailabilityInfoGet(storeId, id2IdNorm[vp.Id], qHtml); _cacheCustom.SaveQty(vp.Id, new QtyItem() { Value = avi.Qty, StoreId = storeId }); } } } List<Variant> validVars = null; if (_ikeaSettings.ProductQtyMin > 0) { validVars = new List<Variant>(); foreach (var v in variants) { if (_cacheCustom.GetQty(v.Id, out List<QtyItem> qtyList)) { int qtySum = qtyList.Sum(q => q.Value); if (qtySum < _ikeaSettings.ProductQtyMin) { _mti.AddLogWarning(string.Format("[VariantsAvailabilityValidation] Variant '{0}' processing interrupt -> current quantity ({1}) < minimum allowable value ({2})", v.Id, qtySum.ToString(), _ikeaSettings.ProductQtyMin.ToString())); _cacheCustom.SaveAsProcessed(v.Id); continue; } } validVars.Add(v); } } else { validVars = variants; } _mti.AddLogFormatted("[VariantsAvailabilityValidation] Finished => valid variants/variants:{0}/{1}.", validVars.Count.ToString(), variants.Count.ToString()); return validVars; } private Dictionary<string, Variant> Ids2VariantsParse(string html, Type type) { _mti.AddLogInfo("<<<<< Ids2VariantsParse started >>>>>"); if (string.IsNullOrEmpty(html)) { return new Dictionary<string, Variant>(); } var ids2Vars = new Dictionary<string, Variant>(); try { if (type == typeof(ModelsIkea.Variations.VariationsSection)) { var j = JsonConvert.DeserializeObject<ModelsIkea.Variations.VariationsSection>(html); foreach (var vrnt in j.variations) { var vrntOptions = vrnt.allOptions.Count > vrnt.options.Count ? vrnt.allOptions : vrnt.options; foreach (var opt in vrntOptions) { if (!string.IsNullOrEmpty(opt.linkId) && !string.IsNullOrEmpty(opt.url)) { ids2Vars[opt.linkId] = new Variant() { Id = opt.linkId, Url = opt.url, IsSelected = opt.isSelected, Price = opt.priceProps.revampPrice.PriceDoubleGet() }; } } } } else if (type == typeof(ModelsIkea.Variations.Variation)) { var json = JObject.Parse(html); var j = JsonConvert.DeserializeObject<ModelsIkea.Variations.Variation>(json["variationStyles"].First.ToString()); var options = j.allOptions.Count > j.options.Count ? j.allOptions : j.options; foreach (var opt in options) { if (!string.IsNullOrEmpty(opt.linkId) && !string.IsNullOrEmpty(opt.url)) { ids2Vars[opt.linkId] = new Variant() { Id = opt.linkId, Url = opt.url, IsSelected = opt.isSelected, Price = opt.priceProps.revampPrice.PriceDoubleGet() }; } } } } catch (Exception ex) { _mti.AddLogError(ex); } _mti.AddLogInfo("<<< variants count: " + ids2Vars.Count.ToString()); return ids2Vars; } private Dictionary<string, string> VariantsIDs2LinksParse(string html, Type type) { _mti.AddLogInfo("<<<<< VariantsIDs2LinksParse started >>>>>"); if (string.IsNullOrEmpty(html)) { return new Dictionary<string, string>(); } var ids2Links = new Dictionary<string, string>(); try { if (type == typeof(ModelsIkea.Variations.VariationsSection)) { var j = JsonConvert.DeserializeObject<ModelsIkea.Variations.VariationsSection>(html); foreach (var vrnt in j.variations) { var vrntOptions = vrnt.allOptions.Count > vrnt.options.Count ? vrnt.allOptions : vrnt.options; foreach (var opt in vrntOptions) { if (!string.IsNullOrEmpty(opt.linkId) && !string.IsNullOrEmpty(opt.url)) { ids2Links[opt.linkId] = opt.url; } } } } else if (type == typeof(ModelsIkea.Variations.Variation)) { var j = JsonConvert.DeserializeObject<ModelsIkea.Variations.Variation>(html); var options = j.allOptions.Count > j.options.Count ? j.allOptions : j.options; foreach (var opt in options) { if (!string.IsNullOrEmpty(opt.linkId) && !string.IsNullOrEmpty(opt.url)) { ids2Links[opt.linkId] = opt.url; } } } } catch (Exception ex) { _mti.AddLogError(ex); } _mti.AddLogInfo("<<< variants IDs2Links count: " + ids2Links.Count.ToString()); return ids2Links; } private HttpMethodParameters _hmpForVariantsLoad = new HttpMethodParameters() { MethodName = "GET", Accept = "application/vnd.ikea.iows+json;version=2.0", httpHeaders = new Dictionary<string, string>() { /* * Headers logic info script. * https://ikeasources.ru/b/m2.prod.js?v=156940268994123778112121 */ //{ "Sec-Fetch-Mode", "cors" }, //{ "Consumer", "DEVRU" }, { "Consumer", "MAMMUT" }, //{ "Contract", "39799" }, { "Contract", "37249" }, { "accept-encoding", "gzip, deflate, br" }, } }; private bool VariantProcessing(Product productForProcessing) { _mti.AddLogInfo("<<<<< VariantProcessing started >>>>>"); _mti.AddLogInfo("<<< product for processing type:" + productForProcessing.GetType()); _mti.AddLogInfo("<<< product for processing id:" + productForProcessing.Art); string artPrepared = productForProcessing.Art.ToUpper(); if (artPrepared.StartsWith("S")) { artPrepared = artPrepared.TrimStart('S'); } if (!_cacheCustom.GetQty(productForProcessing.Art, out List<QtyItem> qtyList)) { qtyList = new List<QtyItem>(); string qtyHtml; _cacheCustom.Load(string.Format("https://api.ingka.ikea.com/cia/availabilities/ru/us?itemNos={0}&expand=StoresList,Restocks", artPrepared), out qtyHtml); foreach (var storeId in _ikeaSettings.StoreIDs) { var avi = ProductAvailabilityInfoGet(storeId, artPrepared, qtyHtml); if (avi.Qty < 1) { avi.Qty = 0; } var qtyItem = new QtyItem() { Value = avi.Qty, StoreId = storeId }; _cacheCustom.SaveQty(productForProcessing.Art, qtyItem); qtyList.Add(qtyItem); } } int qty = qtyList.Sum(q => q.Value); productForProcessing.Quantity = qty.ToString(); foreach (var item in qtyList) { productForProcessing.AddDynamicAttribute($"QTY[{item.StoreId}]", item.Value.ToString()); } _mti.AddLogInfo($"[VariantProcessing] Variant '{productForProcessing.Art}' QTY list:{string.Join("[next]", qtyList.Select(q => q.ToString()))}"); //if (qty == 0) if (_ikeaSettings.ProductQtyMin > 0 && qty < _ikeaSettings.ProductQtyMin) { _mti.AddLogWarning(string.Format("[VariantProcessing] Variant '{0}' processing interrupt -> current quantity ({1}) < minimum allowable value ({2})", productForProcessing.Art, qty, _ikeaSettings.ProductQtyMin)); return true; } /* string productHtml = ""; if (!_url2ProductPageHtml.TryGetValue(productForProcessing.Url, out productHtml)) { var loader = _gps.Proxy.GetHtmlPageLoaderEmpty(); loader.Load(productForProcessing.Url); _url2ProductPageHtml[productForProcessing.Url] = productHtml = loader.Content; } */ //string variantJsonUrl = string.Format("https://iows.ikea.com/retail/iows/us/en/catalog/items/{0},{1}?ignoreErrors=true", pathSegment.ToUpper(), artPrepared); //var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty(); //if (!hpl.Load(variantJsonUrl, true, _hmpForVariantsLoad)) //{ // _mti.AddLogWarning("!!! Can't download variant JSON."); // return false; //} //var json = JObject.Parse(hpl.Content); //var root = json["RetailItemComm"]; //// https://www.ikea.com/us/en/p/hauga-upholstered-bed-frame-lofallet-beige-30490484/ //if (root == null) //{ //ProductPopulateByProductHtml(_url2ProductPageHtml[productForProcessing.Url], productForProcessing); string html; if (!_cacheCustom.Load(productForProcessing.Url, out html)) { _mti.AddLogWarning(string.Format("[VariantProcessing] Variant '{0}' processing interrupt -> can't find product page html.", productForProcessing.Art)); } ProductPopulateByProductHtml(html, productForProcessing); //} //else //{ // var priceToken = root.SelectToken("$.RetailItemCommPriceList.RetailItemCommPrice"); // if (priceToken != null && priceToken.HasValues) // { // if (priceToken is JArray) // { // bool isNewLowerPrice = false; // foreach (var item in priceToken) // { // var price = item["Price"].Value<string>("$"); // var retailPriceType = item["RetailPriceType"].Value<string>("$"); // if (retailPriceType.Equals("RegularSalesUnitPrice", StringComparison.InvariantCultureIgnoreCase)) // { // if (!isNewLowerPrice) // { // var reasonCode = item["ReasonCode"]; // if (reasonCode != null && reasonCode.Value<string>("$").Equals("NewLowerPrice", StringComparison.InvariantCultureIgnoreCase)) // { // isNewLowerPrice = true; // } // productForProcessing.Price = price; // } // } // if (retailPriceType.Equals("IKEAFamilySalesUnitPrice", StringComparison.InvariantCultureIgnoreCase)) // { // productForProcessing.PriceOld = productForProcessing.Price; // productForProcessing.Price = price; // } // } // } // else // { // productForProcessing.Price = priceToken["Price"].Value<string>("$"); // } // } // var optionsToken = root.SelectToken("$.GPRCommSelectionCriteriaSelectionList.GPRCommSelectionCriteriaSelection"); // if (optionsToken != null && optionsToken.HasValues) // { // if (optionsToken is JArray) // { // foreach (var option in optionsToken) // { // var name = option[/*"SelectionCriteriaCode"*/"SelectionCriteriaName"].Value<string>("$"); // var val = option["SelectionCriteriaValue"].Value<string>("$"); // if (!string.IsNullOrEmpty(name) && !string.IsNullOrEmpty(val)) // { // if (val == "-") // { // continue; // } // var nameChars = name.Trim().ToCharArray(); // nameChars[0] = char.ToUpper(nameChars[0]); // productForProcessing.AddDynamicAttribute(new string(nameChars), val.Trim()); // } // } // } // else // { // var name = optionsToken[/*"SelectionCriteriaCode"*/"SelectionCriteriaName"].Value<string>("$"); // var val = optionsToken["SelectionCriteriaValue"].Value<string>("$"); // if (!string.IsNullOrEmpty(name) && !string.IsNullOrEmpty(val)) // { // if (val != "-") // { // var nameChars = name.Trim().ToCharArray(); // nameChars[0] = char.ToUpper(nameChars[0]); // productForProcessing.AddDynamicAttribute(new string(nameChars), val.Trim()); // } // } // } // } // Dictionary<string, string> productMeasuresNames2Values = ProductMeasuresParse(root); // foreach (var n2v in productMeasuresNames2Values) // { // productForProcessing.AddDynamicAttribute(n2v.Key, n2v.Value); // } // //List<PackageLegacy> packages = PackageMeasuresParse(productHtml); // //PackagesAttributesProcessing(productForProcessing, packages); // var productNameTokenTitle2Val = new Dictionary<string, string>(); // var productName = root["ProductName"]; // if (productName != null) // { // productNameTokenTitle2Val["ProductName"] = productName.Value<string>("$").Trim(); // //productForProcessing.Name = productName.Value<string>("$").Trim(); // } // var productTypeName = root["ProductTypeName"]; // if (productTypeName != null) // { // productNameTokenTitle2Val["ProductTypeName"] = productTypeName.Value<string>("$").Trim(); // //productForProcessing.Name += " " + productTypeName.Value<string>("$").Trim(); // } // var validDesignText = root["ValidDesignText"]; // if (validDesignText != null) // { // productNameTokenTitle2Val["ValidDesignText"] = validDesignText.Value<string>("$").Trim(); // } // var itemMeasureReferenceTextMetric = root["ItemMeasureReferenceTextMetric"]; // if (itemMeasureReferenceTextMetric != null) // { // productNameTokenTitle2Val["ItemMeasureReferenceTextMetric"] = itemMeasureReferenceTextMetric.Value<string>("$").Trim(); // } // productForProcessing.Name = productNameTokenTitle2Val["ProductName"] + " " // + string.Join(", ", productNameTokenTitle2Val.Where(kvp => kvp.Key != "ProductName").Select(kvp => kvp.Value)); // productForProcessing.FullDescription = DescriptionFullBuild(root); // //productForProcessing.FullDescription += DescriptionOfPackagesBuild(packages); // /* // var attachmentUrlTokens = root.SelectTokens("$.RetailItemCommAttachmentList..AttachmentUrl"); // if (attachmentUrlTokens != null) // { // var links = from t in attachmentUrlTokens // let url = t.Value<string>("$") // where !string.IsNullOrEmpty(url) // select TextUtils.UrlNewCompose(_domain, url); // if (links.Any()) // { // productForProcessing.AddDynamicAttribute("Инструкция", string.Join(",", links)); // } // } // */ //} if (productForProcessing.ImagesCount == 0) { ImagesFill(productForProcessing); } //var avi = ProductAvailabilityInfoGet(_gps.GrabberSettings.UserParameterGet("StoreId"), artPrepared); //productForProcessing.Quantity = avi.Qty.ToString(); //productForProcessing.AddDynamicAttribute("Коробки", BoxAttributeValueBuild(avi, packages)); return true; } private void ProductPopulateByProductHtml(string productHtml, Product product) { _mti.AddLogFormatted("[ProductPopulateByProductHtml] Started. Art:{0}.", product.Art); if (string.IsNullOrEmpty(productHtml)) { _mti.AddLogWarning("[ProductPopulateByProductHtml] Product html is empty."); return; } product.Name = ProductNameBuild(productHtml); var hDoc = new HtmlDocument(); hDoc.LoadHtml(productHtml); var nodes = hDoc.DocumentNode.SelectNodes(".//div[@data-initial-props]"); if (nodes != null) { var blockClass2InitProps = nodes.ToDictionary(n => n.Attributes["class"].Value, n => n.Attributes["data-initial-props"].Value); foreach (var block in blockClass2InitProps) { var html = HttpUtility.HtmlDecode(block.Value); var j = JObject.Parse(html); if (block.Key.IndexOf("pip-price-package", StringComparison.InvariantCultureIgnoreCase) >= 0) { var price = j["price"]["mainPriceProps"]["price"]; product.Price = string.Format("{0}{1}{2}", price.Value<string>("integer"), j["price"]["mainPriceProps"].Value<string>("separator"), price.Value<string>("decimals")); } else if (block.Key.IndexOf("revamp-product-styles", StringComparison.InvariantCultureIgnoreCase) >= 0) { var variant = JsonConvert.DeserializeObject<ModelsIkea.Variations.Variation>(html); var options = variant.allOptions.Count > variant.options.Count ? variant.allOptions : variant.options; foreach (var opt in options) { if (opt.isSelected && !string.IsNullOrEmpty(opt.title)) { product.AddDynamicAttribute(variant.title, opt.title); } } } else if (block.Key.IndexOf("product-variation-section", StringComparison.InvariantCultureIgnoreCase) >= 0) { var variations = j.SelectToken("variations"); if (variations != null) { variations = variations as JArray; if (variations != null && variations.HasValues) { foreach (JObject v in variations) { var variant = v.ToObject<ModelsIkea.Variations.Variation>(); var options = variant.allOptions.Count > variant.options.Count ? variant.allOptions : variant.options; foreach (var opt in options) { if (opt.isSelected && !string.IsNullOrEmpty(opt.title)) { product.AddDynamicAttribute(variant.title, opt.title); } } } } } } else if (block.Key.IndexOf("product-information-section", StringComparison.InvariantCultureIgnoreCase) >= 0) { var dimName2Val = new Dictionary<string, string>(); try { var dimensionProps = j.Value<JObject>("dimensionProps"); if (dimensionProps != null) { var dims = dimensionProps.Value<JArray>("dimensions"); if (dims != null && dims.HasValues) { foreach (JObject d in dims) { var n = d.Value<string>("name"); var v = d.Value<string>("measure"); if (!string.IsNullOrEmpty(n) && !string.IsNullOrEmpty(v)) { dimName2Val[n.Trim()] = v.Trim(); } } } } } catch { } foreach (var n2v in dimName2Val) { product.AddDynamicAttribute(n2v.Key, n2v.Value); } var pkgs = PackageMeasuresParse(html, _ikeaSettings.Region == Region.EN ? MeasureSystem.US_STANDARD : MeasureSystem.METRIC); if (_ikeaSettings.Region == Region.EN) { var w = PackageHelper.WeightTotalUsStandardValCalc(pkgs); if (_ikeaSettings.PackagesWeightTotalAsAttribute) { product.AddDynamicAttribute("Package weight", string.Format("{0}{1}", w.Key > 0 ? w.Key.ToString() + " lb " : "", w.Value > 0 ? w.Value.ToString() + " oz" : "")); } } else { if (_ikeaSettings.ProductSizesAsPackagesSizesTotal) { product.Weight = PackageHelper.MeasureTotalValCalc(pkgs, MeasureType.WEIGHT, _ikeaSettings.Region).ToString(); product.Height = PackageHelper.MeasureTotalValCalc(pkgs, MeasureType.HEIGHT, _ikeaSettings.Region).ToString(); product.Width = PackageHelper.MeasureTotalValCalc(pkgs, MeasureType.WIDTH, _ikeaSettings.Region).ToString(); product.Depth = PackageHelper.MeasureTotalValCalc(pkgs, MeasureType.LENGTH, _ikeaSettings.Region).ToString(); } } product.FullDescription = DescriptionFullBuild(j.Value<JObject>("productDetailsProps"), dimensions: dimName2Val); } } } } private string ProductNameBuild(string productHtml) { if (!string.IsNullOrEmpty(productHtml)) { var hdoc = new HtmlDocument(); hdoc.LoadHtml(productHtml); //var node_1 = hdoc.DocumentNode.SelectSingleNode(".//div[@id='name']"); //string n1 = NodeValGet(node_1, ".//strong", @"[\r|\t|\n|\s]{2,}[--->] "); //var node_2 = hdoc.DocumentNode.SelectSingleNode(".//h1"); //string n2 = NodeValGet(node_2, ".//strong", @"</div><div[^>]*?>[--->] [next]<[^>]*?>[next]\s{2,}[--->] "); //return string.Format("{0} {1}", n1.Trim(), HttpUtility.HtmlDecode(n2.Trim())).Trim(); var node_2 = hdoc.DocumentNode.SelectSingleNode(".//h1"); var parts = new List<string>(); if (node_2 != null) { foreach (var n in node_2.ChildNodes) { if (!string.IsNullOrEmpty(n.InnerText)) { parts.Add(n.InnerText.Trim()); } } } return HttpUtility.HtmlDecode(string.Join(" ", parts)); } return string.Empty; } private string NodeValGet(HtmlNode node, string removeAllInnerNodesXpath = null, string replaceInnerHtmlRegex = null) { if (node != null) { if (!string.IsNullOrEmpty(removeAllInnerNodesXpath)) { var rems = node.SelectNodes(removeAllInnerNodesXpath); if (rems != null) { foreach (var r in rems) { r.Remove(); } } var val = node.InnerHtml; if (!string.IsNullOrEmpty(replaceInnerHtmlRegex)) { var patterns = UtilSmall.GetParts(replaceInnerHtmlRegex); foreach (var p in patterns) { var fromTo = UtilSmall.GetParts(p, "[--->]"); if (fromTo.Length == 1) { fromTo = new string[2] { fromTo[0], "" }; } val = Regex.Replace(val, fromTo[0], fromTo[1]); } } return val; } } return string.Empty; } private string BoxAttributeValueBuild(AvailabilityInfo avi, List<PackageLegacy> packages, string propDelim = ";", string itemDelim = "[next]") { var vals = from s in avi.SalesLocations join p in packages on s.ItemNo equals p.ItemNo select string.Format("product_set_name:{1}{0}product_count:{2}{0}product_set_ean:{3}{0}row:{4}{0}place:{5}", propDelim, p.ProductName, p.PackagesQty, p.ItemNo, s.Aisle, s.Bin); return string.Join(itemDelim, vals); } private string DescriptionOfPackagesBuild(List<PackageLegacy> packages) { var sBuilder = new StringBuilder(); sBuilder.Append("<div><strong>Packaging</strong></div>"); if (packages.Count > 1) { sBuilder.AppendFormat("<h5>This product comes as {0} packages.</h5>", packages.Sum(p => p.PackagesQty).ToString()); } string idCurrent = "default"; foreach (var pkg in packages) { sBuilder.Append("<div>"); string id = IDCreate(pkg.ItemNo); if (idCurrent != id) { idCurrent = id; sBuilder.AppendFormat("<div><h5>{0}</h5><p>{1}</p><p><span>Article Number</span><br/><span>{2}</span></p></div>", pkg.ProductName, pkg.ProductTypeName, id); } else { sBuilder.Append("<br/>"); } foreach (var measureN2V in pkg.MeasuresName2Val) { sBuilder.AppendFormat("<div>{0}: {1}</div>", PackageHelper.PackageSizeNameTranslate(measureN2V.Key), measureN2V.Value); } sBuilder.AppendFormat("<div>{0}: {1}</div>", "Package(s)", pkg.PackagesQty.ToString()); sBuilder.Append("</div>"); } return sBuilder.ToString(); } private void PackagesAttributesProcessing(Product product, List<PackageLegacy> packages) { /* * http://crm.catalogloader.com/issues/35219 4. Вес товара и габариты упаковки Пояснение: — Кол-во товаров в упаковке сделать характеристикой; Ок. — Расчёт габаритов; Ширина. Берём максимальное значение Высота. Суммируем Длина. Берём максимальное значение Полученное выводим в характеристики. Это будет возможно, если на всех карточках товарах одинаково оформлены габариты Пробуем. Получится — хорошо, не получится — ладно. — Расчёт объёма Рассчитать объём и вывести характеристикой Напишите формулу расчета. Формула расчёта объёма: Полученные габариты суммируем: 58*21*88 = 107 184 см3. Получаем объём в м3. 107 184 * 0,000001 = 0,107184 м3 */ product.AddDynamicAttribute("Кол-во упаковок", packages.Sum(p => p.PackagesQty).ToString()); double weightTotal = PackageHelper.MeasureTotalValCalc(packages, MeasureType.WEIGHT, _ikeaSettings.Region); product.SetAttributeValue("SYS_PRODUCT_WEIGHT", string.Format("{0:N2}", weightTotal)); double heightTotal = PackageHelper.MeasureTotalValCalc(packages, MeasureType.HEIGHT, _ikeaSettings.Region); product.AddDynamicAttribute(PackageHelper.PackageSizeNameTranslate(MeasureType.HEIGHT.ToString()), string.Format("{0:N2}", heightTotal)); double widthMax = PackageHelper.MeasureMaxValueGet(packages, MeasureType.WIDTH, _ikeaSettings.Region); product.AddDynamicAttribute(PackageHelper.PackageSizeNameTranslate(MeasureType.WIDTH.ToString()), string.Format("{0:N2}", widthMax)); double lengthMax = PackageHelper.MeasureMaxValueGet(packages, MeasureType.LENGTH, _ikeaSettings.Region); product.AddDynamicAttribute(PackageHelper.PackageSizeNameTranslate(MeasureType.LENGTH.ToString()), string.Format("{0:N2}", lengthMax)); double pakageVal = heightTotal * widthMax * lengthMax * 0.000001; product.AddDynamicAttribute("Объем, м3", string.Format("{0:N2}", pakageVal)); } private List<PackageLegacy> PackageMeasuresParse(string productHtml, MeasureSystem measureSystem = MeasureSystem.METRIC) { _mti.AddLogInfo("[PackageMeasuresParse] Started."); var packages = new List<PackageLegacy>(); if (!string.IsNullOrEmpty(productHtml)) { try { //string jsonStr = HtmlValueGet(productHtml, ".//div[contains(@class,'js-product-information-section range-revamp-product-information-section')]", "data-initial-props"); string jsonStr = productHtml; if (!string.IsNullOrEmpty(jsonStr)) { jsonStr = HttpUtility.HtmlDecode(jsonStr); var json = JsonConvert.DeserializeObject<ModelsIkea.ProductProps.Props>(jsonStr); var packaging = json.DimensionPropsObject != null && json.DimensionPropsObject.packaging != null ? json.DimensionPropsObject.packaging : null; if (packaging == null) { _mti.AddLogInfo("[PackageMeasuresParse] json.DimensionPropsObject.packaging == null ? TRUE"); packaging = json.productDetailsProps != null && json.productDetailsProps.accordionObject != null ? json.productDetailsProps.accordionObject.packaging : null; if (packaging != null) { _mti.AddLogInfo("[PackageMeasuresParse] packaging IS json.productDetailsProps.accordionObject.packaging"); } } else { _mti.AddLogInfo("[PackageMeasuresParse] packaging IS json.DimensionPropsObject.packaging"); } if (packaging != null && packaging.contentProps != null && packaging.contentProps.packages != null) { foreach (var pck in packaging.contentProps.packages) { if (pck.measurements == null || pck.measurements.Length == 0) { continue; } foreach (var m in pck.measurements) { PackageLegacy package = null; var m2vr = new Dictionary<string, string>(); foreach (var msrm in m) { if (string.IsNullOrEmpty(msrm.label) || string.IsNullOrEmpty(msrm.value)) { continue; } //if (package == null) //{ // package = new PackageLegacy(); //} m2vr[msrm.label.ToUpperInvariant()] = msrm.value; //string valNormal = Regex.Replace(msrm.value, @"\D+?$", ""); //package.MeasuresName2Val[MeasureNameLegacyGet(msrm.label)] = UtilSmall.ConvertToDouble(valNormal); } if (package == null) { package = new PackageLegacy(m2vr, measureSystem); } if (package != null) { package.PackagesQty = pck.quantity.value; package.ItemNo = pck.articleNumber.value; package.ProductName = pck.name; package.ProductTypeName = pck.typeName; packages.Add(package); } } } } else { _mti.AddLogInfo("[PackageMeasuresParse] json.productDetailsProps.accordionObject.packaging == null ? TRUE"); } } } catch (Exception ex) { _mti.AddLogError("PackageMeasuresParse() failed.", ex); } } _mti.AddLogInfo("[PackageMeasuresParse] Finished."); return packages; } private string MeasureNameLegacyGet(string nameModern) { string ptrn = nameModern.ToUpperInvariant(); switch (ptrn) { case "ШИРИНА": return "WIDTH"; case "ВЫСОТА": return "HEIGHT"; case "ДЛИНА": return "LENGTH"; case "ВЕС": return "WEIGHT"; case "ДИАМЕТР": return "DIAMETER"; default: return nameModern.ToUpperInvariant(); } } private string HtmlValueGet(string html, string nodeXpath, string attributeName = null, bool plainTxt = false) { string val = ""; try { var hdoc = new HtmlDocument(); hdoc.LoadHtml(html); var node = hdoc.DocumentNode.SelectSingleNode(nodeXpath); if (node != null) { if (string.IsNullOrEmpty(attributeName)) { val = plainTxt ? node.InnerText : node.OuterHtml; } else { val = node.GetAttributeValue(attributeName, ""); } } } catch (Exception ex) { _mti.AddLogError(ex); } return val; } private void ImagesFill(Product productForProcessing) { string imgsJson = productForProcessing.GetAttributeValue("MEDIA_GRID"); if (string.IsNullOrEmpty(imgsJson)) { string html = ""; /* if (!_url2ProductPageHtml.TryGetValue(productForProcessing.Url, out html)) { var hplImgs = _gps.Proxy.GetHtmlPageLoaderEmpty(); if (hplImgs.Load(productForProcessing.Url)) { _url2ProductPageHtml[productForProcessing.Url] = html = hplImgs.Content; } } */ if (!_cacheCustom.Load(productForProcessing.Url, out html)) { var hplImgs = _gps.Proxy.GetHtmlPageLoaderEmpty(); hplImgs.Load(productForProcessing.Url); } if (!string.IsNullOrEmpty(html)) { var hdoc = new HtmlDocument(); hdoc.LoadHtml(html); //imgsJson = TextUtils.GetHtmlValue(hdoc, ".//div[contains(@class,'js-range-media-grid range-revamp-media-grid')]", "data-initial-props", true, 0); imgsJson = TextUtils.GetHtmlValue(hdoc, ".//div[contains(@class,'js-range-media-grid')]", "data-initial-props", true, 0); } } if (!string.IsNullOrEmpty(imgsJson)) { var imgs = new List<ImageClp>(); var mediaLst = new List<ModelsIkea.Media.MediaListBase>(); try { var j = JsonConvert.DeserializeObject<ModelsIkea.Media.MediaGrid>(imgsJson); mediaLst = j.fullMediaList.Any() ? j.fullMediaList.ToList<ModelsIkea.Media.MediaListBase>() : j.mediaList.ToList<ModelsIkea.Media.MediaListBase>(); } catch (Exception ex) { _mti.AddLogError(ex); } foreach (var media in mediaLst) { if (!media.type.Equals("image", StringComparison.InvariantCultureIgnoreCase) || media.content == null || string.IsNullOrEmpty(media.content.imageFileName) || string.IsNullOrEmpty(media.content.url)) { continue; } imgs.Add(new ImageClp() { FileName = media.content.imageFileName, Url = media.content.url }); } productForProcessing.ImageSafeAdd(imgs); } } private Dictionary<string, string> ProductMeasuresParse(JToken root) { var measuresNames2Vals = new Dictionary<string, string>(); var itemMeasureToken = root.SelectToken("$.RetailItemCommMeasureList.RetailItemCommMeasure"); if (itemMeasureToken != null && itemMeasureToken.HasValues) { if (itemMeasureToken is JArray) { foreach (var pkg in itemMeasureToken) { var nameOrig = pkg["ItemMeasureTypeName"].Value<string>("$"); var val = pkg["ItemMeasureTextMetric"].Value<string>("$"); if (string.IsNullOrEmpty(nameOrig) || string.IsNullOrEmpty(val)) { continue; } measuresNames2Vals[nameOrig.Trim()] = val.Trim(); } } else { var nameOrig = itemMeasureToken["ItemMeasureTypeName"].Value<string>("$"); var val = itemMeasureToken["ItemMeasureTextMetric"].Value<string>("$"); if (!string.IsNullOrEmpty(nameOrig) || !string.IsNullOrEmpty(val)) { measuresNames2Vals[nameOrig.Trim()] = val.Trim(); } } } return measuresNames2Vals; } internal class AvailabilityInfo { internal string ItemNo { get; set; } internal int Qty { get; set; } internal List<SalesLocationInfo> SalesLocations { get; set; } internal AvailabilityInfo() { Qty = -1; SalesLocations = new List<SalesLocationInfo>(); } } internal class SalesLocationInfo { internal string ItemNo { get; set; } internal string Aisle { get; set; } internal string Bin { get; set; } } internal class PackageLegacy { internal Dictionary<string, double> MeasuresName2Val { get; private set; } internal Dictionary<string, string> MeasuresName2ValRaw { get; private set; } internal string ProductName { get; set; } internal string ItemNo { get; set; } internal int PackagesQty { get; set; } public string ProductTypeName { get; internal set; } public MeasureSystem MeasureSystem { get; private set; } internal PackageLegacy(Dictionary<string, string> measuresName2ValRaw, MeasureSystem measureSystem = MeasureSystem.METRIC) { PackagesQty = 1; MeasuresName2Val = new Dictionary<string, double>(); MeasuresName2ValRaw = measuresName2ValRaw; MeasureSystem = measureSystem; Init(); } private void Init() { if (MeasuresName2ValRaw == null) { MeasuresName2ValRaw = new Dictionary<string, string>(); } foreach (var n2rv in MeasuresName2ValRaw) { if (MeasureSystem == MeasureSystem.METRIC) { string valNormal = Regex.Replace(n2rv.Value, @"\D+?$", ""); MeasuresName2Val[n2rv.Key] = UtilSmall.ConvertToDouble(valNormal); } } } internal string ToHtml() { var sBuilder = new StringBuilder("<package>"); foreach (var n2v in MeasuresName2Val) { string nameToLower = n2v.Key.ToLowerInvariant(); double valueNormal = n2v.Value; if (nameToLower != "weight") { valueNormal = valueNormal / 100; } sBuilder.AppendFormat("<{0}>{1}</{0}>", nameToLower, valueNormal.ToString()); } sBuilder.AppendFormat("<{0}>{1}</{0}>", "sku", ItemNo); sBuilder.AppendFormat("<{0}>{1}</{0}>", "quantity", PackagesQty); sBuilder.Append("</package>"); return sBuilder.ToString(); } } internal enum MeasureType { WIDTH, HEIGHT, LENGTH, WEIGHT, DIAMETER } internal enum MeasureSystem { METRIC, US_STANDARD } internal static class PackageHelper { internal static double MeasureMaxValueGet(List<PackageLegacy> packages, MeasureType type, Region r) { double valMax = 0; foreach (var p in packages) { double val; //if (p.MeasuresName2Val.TryGetValue(type.ToString(), out val)) if (p.MeasuresName2Val.TryGetValue(PackageSizeNameTranslate(type, r), out val)) { if (valMax < val) { valMax = val; } } } return valMax; } internal static double MeasureTotalValCalc(List<PackageLegacy> packages, MeasureType type, Region r) { double valTotal = 0; foreach (var p in packages) { double val; //if (p.MeasuresName2Val.TryGetValue(type.ToString(), out val)) if (p.MeasuresName2Val.TryGetValue(PackageSizeNameTranslate(type, r), out val)) { valTotal += val * p.PackagesQty; } } return valTotal; } internal static KeyValuePair<int, int> WeightTotalUsStandardValCalc(List<PackageLegacy> packages) { int lbs = 0; int ozs = 0; string ptrn = @"((?<lb>\d+)\slb)?\s*((?<oz>\d+)\s*oz)?"; foreach (var p in packages) { if (p.MeasureSystem == MeasureSystem.US_STANDARD) { string val; if (p.MeasuresName2ValRaw.TryGetValue(PackageSizeNameTranslate(MeasureType.WEIGHT, Region.EN), out val)) { if (Regex.IsMatch(val, ptrn, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase)) { var grps = Regex.Match(val, ptrn, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase).Groups; if (grps["lb"] != null && !string.IsNullOrEmpty(grps["lb"].Value)) { lbs += int.Parse(grps["lb"].Value); } if (grps["oz"] != null && !string.IsNullOrEmpty(grps["oz"].Value)) { ozs += int.Parse(grps["oz"].Value); lbs += ozs / 16; // 1 lb = 16 oz ozs %= 16; } } } } } return new KeyValuePair<int, int>(lbs, ozs); } internal static string PackageSizeNameTranslate(string source) { if (string.IsNullOrEmpty(source)) { return string.Empty; } switch (source) { case "WEIGHT": return "Вес упаковки, кг"; case "HEIGHT": return "Высота упаковки, см"; case "LENGTH": return "Длина упаковки, см"; case "WIDTH": return "Ширина упаковки, см"; case "DIAMETER": return "Диаметр, см"; default: return string.Empty; } } private static Dictionary<Region, Dictionary<MeasureType, string>> _region2Measure2Type = new Dictionary<Region, Dictionary<MeasureType, string>>() { { Region.PL, new Dictionary<MeasureType, string>() { { MeasureType.WEIGHT, "WAGA" }, { MeasureType.WIDTH, "SZEROKOŚĆ" }, { MeasureType.DIAMETER, "ŚREDNICA" }, { MeasureType.HEIGHT, "WYSOKOŚĆ" }, { MeasureType.LENGTH, "DŁUGOŚĆ" } } }, { Region.EN, new Dictionary<MeasureType, string>() { { MeasureType.WEIGHT, "WEIGHT" }, { MeasureType.WIDTH, "WIDTH" }, { MeasureType.DIAMETER, "DIAMETER" }, { MeasureType.HEIGHT, "HEIGHT" }, { MeasureType.LENGTH, "LENGTH" } } }, { Region.RU, new Dictionary<MeasureType, string>() { { MeasureType.WEIGHT, "ВЕС" }, { MeasureType.WIDTH, "ШИРИНА" }, { MeasureType.DIAMETER, "ДИАМЕТР" }, { MeasureType.HEIGHT, "ВЫСОТА" }, { MeasureType.LENGTH, "ДЛИННА" } } } }; internal static string PackageSizeNameTranslate(MeasureType mt, Region r) { Dictionary<MeasureType, string> measure2LocalName; if (_region2Measure2Type.TryGetValue(r, out measure2LocalName)) { string localName; if (measure2LocalName.TryGetValue(mt, out localName)) { return localName; } } return string.Empty; } } private Dictionary<string, double> RetailItemCommPackageMeasureParse(JToken root) { var measureName2Val = new Dictionary<string, double>(); if (root is JArray) { foreach (var pkg in root) { var nameOrig = pkg["PackageMeasureType"].Value<string>("$"); var val = pkg["PackageMeasureTextMetric"].Value<string>("$"); val = Regex.Replace(val, @"\D+?$", ""); if (string.IsNullOrEmpty(nameOrig) || string.IsNullOrEmpty(val)) { continue; } measureName2Val[nameOrig] = UtilSmall.ConvertToDouble(val); } } else { var nameOrig = root["PackageMeasureType"].Value<string>("$"); var val = root["PackageMeasureTextMetric"].Value<string>("$"); val = Regex.Replace(val, @"\D+?$", ""); if (!string.IsNullOrEmpty(nameOrig) || !string.IsNullOrEmpty(val)) { measureName2Val[nameOrig] = UtilSmall.ConvertToDouble(val); } } return measureName2Val; } private string DescriptionFullBuild(JToken root, string packagesHtml = null, Dictionary<string, string> dimensions = null) { var sBuilder = new StringBuilder(); JObject detailes = null; var benefitsToken = root.SelectToken("$.RetailItemCustomerBenefitList.RetailItemCustomerBenefit"); if (benefitsToken != null && benefitsToken.HasValues) { sBuilder.Append("<div><strong>Product details</strong></div>"); if (benefitsToken is JArray) { foreach (var b in benefitsToken) { sBuilder.AppendFormat("<div>{0}</div>", b["CustomerBenefitText"].Value<string>("$")); } } else { sBuilder.AppendFormat("<div>{0}</div>", benefitsToken["CustomerBenefitText"].Value<string>("$")); } } else { detailes = root.Value<JObject>("productDescriptionProps"); if (detailes != null) { sBuilder.Append("<div><strong>Product details</strong></div>"); var prfs = detailes.Value<JArray>("paragraphs"); if (prfs != null && prfs.HasValues) { foreach (var pr in prfs) { sBuilder.AppendFormat("<div>{0}</div>", pr.Value<string>()); } } var disr = detailes.Value<string>("designerName"); if (!string.IsNullOrEmpty(disr)) { sBuilder.AppendFormat("<div><strong>Designer</strong></div><div>{0}</div>", disr); } } } var designerObj = root.SelectToken("$.DesignerNameComm"); if (designerObj != null) { sBuilder.AppendFormat("<div><strong>Designer</strong></div><div>{0}</div>", designerObj.Value<string>("$")); } if (packagesHtml != null) { sBuilder.Append(packagesHtml); } var itemMeasureToken = root.SelectToken("$.RetailItemCommMeasureList.RetailItemCommMeasure"); if (itemMeasureToken != null && itemMeasureToken.HasValues) { sBuilder.Append("<div><strong>Product size</strong></div>"); if (itemMeasureToken is JArray) { foreach (var pkg in itemMeasureToken) { var nameOrig = pkg["ItemMeasureTypeName"].Value<string>("$"); var val = pkg["ItemMeasureTextMetric"].Value<string>("$"); if (string.IsNullOrEmpty(nameOrig) || string.IsNullOrEmpty(val)) { continue; } sBuilder.AppendFormat("<div>{0}: {1}</div>", Translate(nameOrig), val); } } else { var nameOrig = itemMeasureToken["ItemMeasureTypeName"].Value<string>("$"); var val = itemMeasureToken["ItemMeasureTextMetric"].Value<string>("$"); if (!string.IsNullOrEmpty(nameOrig) || !string.IsNullOrEmpty(val)) { sBuilder.AppendFormat("<div>{0}: {1}</div>", Translate(nameOrig), val); } } } else if (dimensions != null) { foreach (var d in dimensions) { sBuilder.AppendFormat("<div>{0}: {1}</div>", d.Key, d.Value); } } JObject accordion = detailes != null ? root.Value<JObject>("accordionObject") : null; var environmentTokens = root.SelectTokens("$.RetailItemCustomerEnvironmentList..RetailItemEnvironmentText..EnvironmentText"); if (environmentTokens != null) { sBuilder.Append("<div><strong>Environment</strong></div>"); foreach (var e in environmentTokens) { sBuilder.AppendFormat("<div>{0}</div>", e.Value<string>("$")); } } else if (detailes != null) { if (accordion != null) { var envData = accordion.SelectToken("sustainabilityAndEnvironment.contentProps.environmentalData"); if (envData != null) { var envs = envData as JArray; if (envs != null && envs.HasValues) { sBuilder.Append("<div><strong>Sustainability & environment</strong></div>"); foreach (var e in environmentTokens) { var txts = e.Value<JArray>("texts"); if (txts != null && txts.HasValues) { foreach (var t in txts) { sBuilder.AppendFormat("<div>{0}</div>", t.Value<string>()); } } } } } } } sBuilder.Append("<div><strong>Materials & care</strong></div>"); var materialsRoot = root.SelectToken("$.RetailItemCustomerMaterialList.RetailItemCustomerMaterial"); if (materialsRoot == null && accordion != null) { var m = accordion.SelectToken("materialsAndCare.contentProps.materials"); if (m != null && m.HasValues) { materialsRoot = m.First.SelectToken("materials"); } } string materialsStr = MaterialStringBuild(materialsRoot); sBuilder.Append(materialsStr); var caresToken = root.SelectToken("$.RetailItemCareInstructionList.RetailItemCareInstruction"); if (caresToken != null) { //sBuilder.Append("<div><strong>Инструкция по уходу</strong></div>"); if (caresToken is JArray) { foreach (var item in caresToken) { var caresHeaderObj = item["CareInstructionHeader"]; if (caresHeaderObj != null && caresHeaderObj.HasValues) { sBuilder.AppendFormat("<h5>{0}</h5>", caresHeaderObj.Value<string>("$")); } var caresTexts = item.SelectTokens("$..CareInstructionText"); if (caresTexts != null) { foreach (var txt in caresTexts) { sBuilder.AppendFormat("<div>{0}</div>", txt.Value<string>("$")); } } } } else { var caresHeaderObj = caresToken["CareInstructionHeader"]; if (caresHeaderObj != null && caresHeaderObj.HasValues) { sBuilder.AppendFormat("<h5>{0}</h5>", caresHeaderObj.Value<string>("$")); } var caresTexts = caresToken.SelectTokens("$..CareInstructionText"); if (caresTexts != null) { foreach (var txt in caresTexts) { sBuilder.AppendFormat("<div>{0}</div>", txt.Value<string>("$")); } } } } else if (accordion != null) { var careInstructions = accordion.SelectToken("materialsAndCare.contentProps.careInstructions"); if (careInstructions != null && careInstructions.HasValues) { var txts = careInstructions.First().Value<JArray>("texts"); if (txts != null && txts.HasValues) { foreach (var t in txts) { sBuilder.AppendFormat("<div>{0}</div>", t); } } } } return sBuilder.ToString(); } private string MaterialStringBuild(JToken materialsRoot) { var sBuilder = new StringBuilder(); if (materialsRoot != null && materialsRoot.HasValues) { //sBuilder.Append("<div><strong>Материалы</strong></div>"); if (materialsRoot is JArray) { foreach (var item in materialsRoot) { string materialTokenString = MaterialTokenToString(item); sBuilder.Append(materialTokenString); } } else { string materialTokenString = MaterialTokenToString(materialsRoot); sBuilder.Append(materialTokenString); } } return sBuilder.ToString(); } private string MaterialTokenToString(JToken t) { var sBuilder = new StringBuilder(); var obj = t as JObject; if (obj != null && obj.HasValues) { var productTypeText = obj["ProductTypeText"]; if (productTypeText != null) { sBuilder.AppendFormat("<div>{0}</div>", productTypeText.Value<string>("$")); } var retailItemPartMaterial = obj.SelectToken("$..RetailItemPartMaterial"); if (retailItemPartMaterial != null && retailItemPartMaterial.HasValues) { if (retailItemPartMaterial is JArray) { foreach (var item in retailItemPartMaterial) { var materialText = item["MaterialText"]; var partText = item["PartText"]; if (partText != null) { sBuilder.AppendFormat("<div>{0} {1}</div>", partText.Value<string>("$"), materialText.Value<string>("$")); } else { sBuilder.AppendFormat("<div>{0}</div>", materialText.Value<string>("$")); } } } else { var materialText = retailItemPartMaterial["MaterialText"]; var partText = retailItemPartMaterial["PartText"]; if (partText != null) { sBuilder.AppendFormat("<div>{0} {1}</div>", partText.Value<string>("$"), materialText.Value<string>("$")); } else { sBuilder.AppendFormat("<div>{0}</div>", materialText.Value<string>("$")); } } } else { string part = obj.Value<string>("part"); string material = obj.Value<string>("material"); if (!string.IsNullOrEmpty(part) && !string.IsNullOrEmpty(material)) { sBuilder.AppendFormat("<div>{0} {1}</div>", part, material); } else if (!string.IsNullOrEmpty(material)) { sBuilder.AppendFormat("<div>{0}</div>", material); } } } return sBuilder.ToString(); } private string Translate(string nameOrig) { if (nameOrig.Equals("WIDTH")) { return "Width"; } else if (nameOrig.Equals("HEIGHT")) { return "Height"; } else if (nameOrig.Equals("LENGTH")) { return "Length"; } else if (nameOrig.Equals("WEIGHT")) { return "Weight"; } else { return nameOrig; } } internal class Variant { //public string DataLinkId { get; set; } //public string DataUrl { get; set; } public string Id { get; set; } public string Url { get; set; } public bool IsSelected { get; set; } public double Price { get; set; } } private void ProductImagesNumberTruncate(Product p, int maxImgsNumber) { var imgs = p.ImagesGet(); if (imgs.Count < maxImgsNumber + 1) { return; } p.ImagesClear(); for (int i = 0; i < maxImgsNumber; i++) { p.ImageAdd(imgs.ElementAt(i).Key, imgs.ElementAt(i).Value); } } private Product HtmlBlocksClear(Product p) { if (p == null) { return null; } if (!string.IsNullOrEmpty(p.HtmlBlockProperties)) { p.HtmlBlockProperties = ""; } if (!string.IsNullOrEmpty(p.HtmlBlockCombinations)) { p.HtmlBlockCombinations = ""; } if (!string.IsNullOrEmpty(p.GetAttributeValue("SYS_IMAGES_ALL"))) { p.SetAttributeValue("SYS_IMAGES_ALL", ""); } if (!string.IsNullOrEmpty(p.GetAttributeValue("SYS_ALL_COMB_ATTR"))) { p.SetAttributeValue("SYS_ALL_COMB_ATTR", ""); } if (!string.IsNullOrEmpty(p.GetAttributeValue("SYS_COMB_DATA"))) { p.SetAttributeValue("SYS_COMB_DATA", ""); } if (!string.IsNullOrEmpty(p.GetAttributeValue("PRICE_FAMILY"))) { p.SetAttributeValue("PRICE_FAMILY", ""); } if (!string.IsNullOrEmpty(p.GetAttributeValue("PRODUCT_STYLES"))) { p.SetAttributeValue("PRODUCT_STYLES", ""); } if (!string.IsNullOrEmpty(p.GetAttributeValue("VARIATION_SECTION"))) { p.SetAttributeValue("VARIATION_SECTION", ""); } return p; } private List<Product> ProductsFromCombinationsGet(Product mainProduct) { var mods = new List<Product>(); if (mainProduct == null) { return mods; } foreach (var c in mainProduct.CombinationsGet()) { var mod = mainProduct.Clone(); mod.Category = mainProduct.Category; mod.FullDescription = c.FullDescription; mod.Name = c.Name; mod.Url = c.Url; //mod.SetAttributeValue("SYS_PRODUCT_WEIGHT", c.GetAttributeValue("SYS_PRODUCT_WEIGHT")); mod.CombinationsRemove(); mod.DynamicAttributesClear(); foreach (var attr in c.GetDynamicAttributes()) { //if (attr.Key.Contains("Вес упаковки")) //{ // mod.SetAttributeValue("SYS_PRODUCT_WEIGHT", c.GetAttributeValue(attr.Value)); //} mod.AddDynamicAttribute(attr.Key, c.GetAttributeValue(attr.Value)); } mod.SetAttributeValue("SYS_PRODUCT_WEIGHT", c.Weight); mod.SetAttributeValue("SYS_PRODUCT_SKU", c.Art); mod.SetAttributeValue("SYS_PRODUCT_ART", c.Art); mod.Quantity = c.Quantity; mod.Price = c.Price; mod.PriceOld = c.PriceOld; mod.ImagesClear(); foreach (var img in c.ImagesGet()) { mod.ImageAdd(img.Key, img.Value); } mods.Add(mod); } return mods; } private string IDCreate(string sourceID) { if (string.IsNullOrEmpty(sourceID)) { return sourceID; } sourceID = Regex.Replace(sourceID, @"\D", ""); var arr = sourceID.ToCharArray(); var newChars = new List<char>(); for (int i = 0; i < arr.Length; i++) { if (i > 0 && i % 3 == 0) { newChars.Add('.'); } newChars.Add(arr[i]); } string newID = new string(newChars.ToArray()); if (!string.IsNullOrEmpty(newID)) { newID = newID.TrimEnd('.'); } return newID; } private bool IsDuplicateCategory(Combination cmb, List<int> propsHashes) { var attrs = cmb.GetDynamicAttributes(); if (attrs.Count < 1) { return false;//return true; } var strBuilder = new StringBuilder(); foreach (var attr in attrs) { strBuilder.Append(cmb.GetAttributeValue(attr.Value)); } int hash = strBuilder.ToString() .GetHashCode(); if (propsHashes.Contains(hash)) { return true; } propsHashes.Add(hash); return false; } private HttpMethodParameters _hmpForQuantity = new HttpMethodParameters() { MethodName = "GET", Accept = "application/json;version=2", //Accept = "application/vnd.ikea.iows+json;version=2.0", httpHeaders = new Dictionary<string, string>() { /* * https://www.ikea.com/ru/ru/products/javascripts/range-pip-main.751a00ccf1e16b028440.js * "x-client-id" прописан хардкодом в скрипте. * В один прекрасный момент, все может измениться :) */ { "X-Client-Id", "b6c117e5-ae61-4ef5-b4cc-e0b1e37f0631" }, { "accept-encoding", "gzip, deflate, br" }, /* * https://www.ikea.com/ext/us/task-rabbit/js/script.min.js {"Consumer", "MAMMUT"}, {"Contract", "37249"}, */ } }; private int ProductQuantityGet(string shopCode, string productArt) { int qty = -1; _mti.AddLogFormatted("[ProductQuantityGet] Started. Shop code:'{0}'.", shopCode); var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty(); if (hpl.Load(string.Format("https://api.ingka.ikea.com/cia/availabilities/ru/us?itemNos={0}&expand=StoresList,Restocks", productArt), false, _hmpForQuantity)) { try { var availObject = JsonConvert.DeserializeObject<ModelsIkea.Availability.AvailabilityObject>(hpl.Content); var shopAvailObj = availObject.availabilities.FirstOrDefault(av => av.classUnitKey.classUnitCode == shopCode); if (shopAvailObj == null) { _mti.AddLogFormatted("[ProductQuantityGet] Can't find availability info for shop code:'{0}'.", shopCode); } else { qty = shopAvailObj.buyingOption.cashCarry.availability != null ? shopAvailObj.buyingOption.cashCarry.availability.quantity : 0; } } catch (Exception ex) { _mti.AddLogError("Availability parsing failed.", ex); throw; } } _mti.AddLogFormatted("[ProductQuantityGet] Finished. Shop code:'{0}' -> Quantity:{1}.", shopCode, qty.ToString()); return qty; } private AvailabilityInfo ProductAvailabilityInfoGet(string shopCode, string productArt, string html = null) { AvailabilityInfo avi = new AvailabilityInfo() { ItemNo = productArt }; _mti.AddLogFormatted("[ProductAvailabilityInfoGet] Started. Product Art:'{0}', Shop code:'{1}', qty html length:{2}.", productArt, shopCode, !string.IsNullOrEmpty(html) ? html.Length.ToString() : "0"); if (string.IsNullOrEmpty(html)) { var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty(); hpl.Load(string.Format(_ikeaSettings.RegionSettings.ProductAvailabilityUrlTemplate, productArt), true, _hmpForQuantity); html = hpl.Content; } //if (hpl.Load(string.Format("https://api.ingka.ikea.com/cia/availabilities/ru/us?itemNos={0}&expand=StoresList,Restocks,SalesLocations", productArt), false, _hmpForQuantity)) try { var availObject = JsonConvert.DeserializeObject<ModelsIkea.Availability.AvailabilityObject>(html); ModelsIkea.Availability.Availability shopAvailObj = null; ModelsIkea.Availability.AvailabilityDataItem shopAvailData = null; if (availObject.availabilities != null) { shopAvailObj = availObject.availabilities.FirstOrDefault(av => av.classUnitKey.classUnitCode == shopCode); } if (shopAvailObj == null) { if (availObject.data != null) { shopAvailData = availObject.data.FirstOrDefault(d => d.classUnitKey.classUnitCode == shopCode && d.isInCashAndCarryRange.HasValue && d.isInCashAndCarryRange.Value); if (shopAvailData != null) { avi.Qty = shopAvailData.availableStocks.First(s => s._type == "CASHCARRY").quantity; } } } else { avi.Qty = shopAvailObj.buyingOption.cashCarry.availability != null ? shopAvailObj.buyingOption.cashCarry.availability.quantity : 0; } if (shopAvailObj == null && shopAvailData == null) { _mti.AddLogFormatted("[ProductAvailabilityInfoGet] Can't find availability info for product art:'{0}' and shop code:'{1}'.", productArt, shopCode); } /* var salesLocationsObj = availObject.salesLocations.FirstOrDefault(sl => sl.classUnitKey.classUnitCode == shopCode); if (salesLocationsObj == null) { _mti.AddLogFormatted("[ProductAvailabilityInfoGet] Can't find sales locations info for shop code:'{0}'.", shopCode); } else { if (salesLocationsObj.childItems != null) { var salesLocations = salesLocationsObj.childItems .Where(ch => ch.salesLocations != null && ch.salesLocations.Length > 0); if (salesLocations.Any()) { foreach (var ch in salesLocations) { if (ch.salesLocations[0].aisleAndBin != null) { avi.SalesLocations.Add(new SalesLocationInfo() { ItemNo = IDCreate(ch.itemKey.itemNo), Aisle = ch.salesLocations[0].aisleAndBin.aisle, Bin = ch.salesLocations[0].aisleAndBin.bin }); } } } } else { avi.SalesLocations = salesLocationsObj.salesLocations .Where(s => s != null && s.aisleAndBin != null) .Select(s => new SalesLocationInfo() { ItemNo = IDCreate(salesLocationsObj.itemKey.itemNo), Aisle = s.aisleAndBin.aisle, Bin = s.aisleAndBin.bin }).ToList(); } } */ } catch (Exception ex) { _mti.AddLogError("Availability parsing failed.", ex); throw; } _mti.AddLogFormatted("[ProductAvailabilityInfoGet] Finished. Product Art:'{0}', Shop code:'{1}' -> Quantity:{2}, SalesLocations count:{3}.", productArt, shopCode, avi.Qty.ToString(), avi.SalesLocations.Count.ToString()); return avi; } private void ProductImagesRename(Product p) { var imgs = p.ImagesGet(); p.ImagesClear(); int idx = 1; var newImgs = new Dictionary<string, string>(); foreach (var img in imgs) { newImgs[img.Key] = MakeImgName2Sku(img.Key, idx, p.Art.Replace(".", ""));//product.ID); idx++; } p.ImageAdd(newImgs); } private string MakeImgName2Sku(string imgHref, int idx, string source) { var expanImg = new Regex(@"\.(jpg|png|bmp|gif|jpeg)", RegexOptions.IgnoreCase).Match(imgHref).Value; if (string.IsNullOrEmpty(expanImg)) { expanImg = ".jpg"; } if (idx > 1) { return source + "-" + idx.ToString(CultureInfo.InvariantCulture) + expanImg; } return source + expanImg; } private Dictionary<string, double> PackageSizesGet(string html) { string pattern = @"var jProductData = (\{[\w\W]*?\});"; if (string.IsNullOrEmpty(html) || !Regex.IsMatch(html, pattern) || string.IsNullOrEmpty(_product.Url)) { return new Dictionary<string, double>(); } string json = Regex.Match(html, pattern).Groups[1].Value; var sizeName2Value = new Dictionary<string, double>(); try { string partNumber = new Uri(_product.Url).Segments.Last().TrimEnd(new[] { '/' }); var jObj = JObject.Parse(json); var mainProdJtoken = jObj["product"]["items"].FirstOrDefault(j => ((string)j["partNumber"]).Equals(partNumber)); if (mainProdJtoken == null) { return new Dictionary<string, double>(); } JArray pkgInfoArr = mainProdJtoken["pkgInfoArr"] as JArray; if (pkgInfoArr == null) { return new Dictionary<string, double>(); } double maxWidth = 0; double maxLength = 0; double packagesQty = 0; foreach (var pkgInfo in pkgInfoArr.SelectTokens("$..pkgInfo")) { if (!_mti.CanContinue()) { break; } foreach (var item in pkgInfo.Children()) { var qtyProp = item["quantity"]; if (qtyProp == null) { continue; } int qty = (int)qtyProp; packagesQty += qty; foreach (var prop in item.Children<JProperty>()) { string valueStr = (string)prop.Value; if (string.IsNullOrEmpty(valueStr)) { continue; } valueStr = Regex.Replace(valueStr, @"(?<=\d)\s.*", "") .Replace(".", ","); double value; if (!double.TryParse(valueStr, out value)) { continue; } string name = prop.Name.Trim(); if (name.Equals("widthMet", StringComparison.InvariantCultureIgnoreCase)) { maxWidth = (maxWidth > value) ? maxWidth : value; continue; } if (name.Equals("lengthMet", StringComparison.InvariantCultureIgnoreCase)) { maxLength = (maxLength > value) ? maxLength : value; continue; } if (sizeName2Value.Keys.Contains(prop.Name.Trim())) { sizeName2Value[prop.Name.Trim()] += value * qty; } else { sizeName2Value[prop.Name.Trim()] = value; } } } } sizeName2Value["widthMet"] = maxWidth; sizeName2Value["lengthMet"] = maxLength; sizeName2Value["packagesQty"] = packagesQty; for (int i = 0; i < sizeName2Value.Count; i++) { sizeName2Value[sizeName2Value.ElementAt(i).Key] = Math.Round(sizeName2Value.ElementAt(i).Value, 2); } } catch (Exception ex) { _mti.AddLogError("<<<<< PackageSizesGet() failed.", ex); } return sizeName2Value; } private Dictionary<string, double> PackageSizesGet(JObject jObj, string partNumber) { if (string.IsNullOrEmpty(partNumber) || jObj == null) { return new Dictionary<string, double>(); } var sizeName2Value = new Dictionary<string, double>(); try { var mainProdJtoken = jObj["product"]["items"].FirstOrDefault(j => ((string)j["partNumber"]).Equals(partNumber)); if (mainProdJtoken == null) { return new Dictionary<string, double>(); } JArray pkgInfoArr = mainProdJtoken["pkgInfoArr"] as JArray; if (pkgInfoArr == null) { return new Dictionary<string, double>(); } double maxWidth = 0; double maxLength = 0; double packagesQty = 0; foreach (var pkgInfo in pkgInfoArr.SelectTokens("$..pkgInfo")) { if (!_mti.CanContinue()) { break; } foreach (var item in pkgInfo.Children()) { var qtyProp = item["quantity"]; if (qtyProp == null) { continue; } int qty = (int)qtyProp; packagesQty += qty; foreach (var prop in item.Children<JProperty>()) { string valueStr = (string)prop.Value; if (string.IsNullOrEmpty(valueStr)) { continue; } valueStr = Regex.Replace(valueStr, @"(?<=\d)\s.*", "") .Replace(".", ","); double value; if (!double.TryParse(valueStr, out value)) { continue; } string name = prop.Name.Trim(); if (name.Equals("widthMet", StringComparison.InvariantCultureIgnoreCase)) { maxWidth = (maxWidth > value) ? maxWidth : value; continue; } if (name.Equals("lengthMet", StringComparison.InvariantCultureIgnoreCase)) { maxLength = (maxLength > value) ? maxLength : value; continue; } if (sizeName2Value.Keys.Contains(prop.Name.Trim())) { sizeName2Value[prop.Name.Trim()] += value * qty; } else { sizeName2Value[prop.Name.Trim()] = value; } } } } sizeName2Value["widthMet"] = maxWidth; sizeName2Value["lengthMet"] = maxLength; sizeName2Value["packagesQty"] = packagesQty; for (int i = 0; i < sizeName2Value.Count; i++) { sizeName2Value[sizeName2Value.ElementAt(i).Key] = Math.Round(sizeName2Value.ElementAt(i).Value, 2); } } catch (Exception ex) { _mti.AddLogError("<<<<< PackageSizesGet() failed.", ex); } return sizeName2Value; } private void Init(GrabCatalogBuildScriptParameters p) { _mti = p.Process.m_ti; _category = new Category { ID = "0" }; } } namespace ModelsIkea { namespace Info { public class InformationSection { public bool isClient { get; set; } public Dimensionprops dimensionProps { get; set; } public Productdetailsprops productDetailsProps { get; set; } public object productReviewsProps { get; set; } public bool technicalInformationProps { get; set; } } public class Dimensionprops { public string title { get; set; } public string subtitle { get; set; } public Dimension[] dimensions { get; set; } public object[] images { get; set; } public Fallbackimage fallbackImage { get; set; } public string eventLabel { get; set; } } public class Fallbackimage { public string alt { get; set; } public string id { get; set; } public string imageFileName { get; set; } public string url { get; set; } public string type { get; set; } } public class Dimension { public string type { get; set; } public string name { get; set; } public string measure { get; set; } } public class Productdetailsprops { public string title { get; set; } public string eventLabel { get; set; } public Productdescriptionprops productDescriptionProps { get; set; } public Accordionobject accordionObject { get; set; } } public class Productdescriptionprops { public string[] paragraphs { get; set; } public string designerLabel { get; set; } public string designerName { get; set; } } public class Accordionobject { public Materialsandcare materialsAndCare { get; set; } public object sustainabilityAndEnvironment { get; set; } public Packaging packaging { get; set; } public Assemblyanddocuments assemblyAndDocuments { get; set; } } public class Materialsandcare { public string id { get; set; } public string title { get; set; } public Contentprops contentProps { get; set; } } public class Contentprops { public Material[] materials { get; set; } public Careinstruction[] careInstructions { get; set; } public object[] proposition65 { get; set; } public Package[] packages { get; set; } public string totalNoOfPackagesText { get; set; } public Attachments attachments { get; set; } } public class Material { public string productType { get; set; } //public Material1[] materials { get; set; } public Material[] materials { get; set; } public string material { get; set; } public string part { get; set; } } //public class Material1 //{ // public string material { get; set; } // public string part { get; set; } //} public class Careinstruction { public string[] texts { get; set; } } public class Packaging { public string id { get; set; } public string title { get; set; } //public Contentprops1 contentProps { get; set; } public Contentprops contentProps { get; set; } } //public class Contentprops1 //{ // public Package[] packages { get; set; } // public string totalNoOfPackagesText { get; set; } //} public class Package { public string name { get; set; } public string typeName { get; set; } public Articlenumber articleNumber { get; set; } public Measurement[][] measurements { get; set; } public Quantity quantity { get; set; } public bool multiPackDisclaimerText { get; set; } } public class Articlenumber { public string label { get; set; } public string value { get; set; } } public class Quantity { public string label { get; set; } public int value { get; set; } } public class Measurement { public string label { get; set; } public string value { get; set; } } public class Assemblyanddocuments { public string id { get; set; } public string title { get; set; } //public Contentprops2 contentProps { get; set; } public Contentprops contentProps { get; set; } } //public class Contentprops2 //{ // public Attachments attachments { get; set; } //} public class Attachments { public Assembly assembly { get; set; } public Software software { get; set; } public Other other { get; set; } public Disassembly disassembly { get; set; } } public class Assembly { public string header { get; set; } public Attachment[] attachments { get; set; } } public class Attachment { public string id { get; set; } public string url { get; set; } public string label { get; set; } } public class Software { public string header { get; set; } public object[] attachments { get; set; } } public class Other { public string header { get; set; } public object[] attachments { get; set; } } public class Disassembly { public string header { get; set; } public object[] attachments { get; set; } } } namespace Price { public class PricePackage { public object newProductText { get; set; } public string newLowerPriceText { get; set; } public object familyText { get; set; } public object timeRestrictedOfferText { get; set; } public string productName { get; set; } public string productDescription { get; set; } public string measurementText { get; set; } public Price price { get; set; } public object productReviewsProps { get; set; } public object productDiscountsProps { get; set; } public object ecoWeeeFees { get; set; } public object timeFrame { get; set; } public object vatInfoText { get; set; } public object energyLabelProps { get; set; } public string ecoWeeeFeeExcludeText { get; set; } public string totalPriceText { get; set; } public string ecoWeeeFeeLinkText { get; set; } public object ecoWeeeFeeLink { get; set; } public object validToFromText { get; set; } public object validToFromAndWhileSupplyLastText { get; set; } public object whileSupplyLastText { get; set; } } public class Price { public Mainpriceprops mainPriceProps { get; set; } public Previouspriceprops previousPriceProps { get; set; } public string previousPriceText { get; set; } public object comparisonPriceText { get; set; } public string discount { get; set; } } public class Mainpriceprops { public Price1 price { get; set; } public string unit { get; set; } public string currencySymbol { get; set; } public string separator { get; set; } public bool hasTrailingCurrency { get; set; } public bool hasHighlight { get; set; } public bool hasStrikeThrough { get; set; } public bool isSmall { get; set; } public bool isSecondary { get; set; } } public class Price1 { public string integer { get; set; } public string decimals { get; set; } } public class Previouspriceprops { //public Price2 price { get; set; } public Price1 price { get; set; } public string unit { get; set; } public string currencySymbol { get; set; } public string separator { get; set; } public bool hasTrailingCurrency { get; set; } public bool hasHighlight { get; set; } public bool hasStrikeThrough { get; set; } public bool isSmall { get; set; } public bool isSecondary { get; set; } } //public class Price2 //{ // public string integer { get; set; } // public string decimals { get; set; } //} } namespace Variations { public class VariationsSection { public List<Variation> variations { get; set; } public bool disabled { get; set; } public VariationsSection() { variations = new List<Variation>(); } } public class Variation { public string title { get; set; } public string selectedOption { get; set; } public List<Option> allOptions { get; set; } // akl_02112021: a new field of variations appeared. public List<Option> options { get; set; } public string code { get; set; } public Variation() { options = new List<Option>(); allOptions = new List<Option>(); } } public class Option { public string url { get; set; } public string title { get; set; } public bool isSelected { get; set; } public string linkId { get; set; } public Image image { get; set; } public Priceprops priceProps { get; set; } } public class Image { public string alt { get; set; } public string id { get; set; } public string imageFileName { get; set; } public string url { get; set; } public string type { get; set; } } public class Priceprops { public string priceNumeral { get; set; } public Revampprice revampPrice { get; set; } public Priceformat priceFormat { get; set; } } public class Revampprice { public int numDecimals { get; set; } public string separator { get; set; } public string integer { get; set; } public string decimals { get; set; } public string currencySymbol { get; set; } public string currencyPrefix { get; set; } public string currencySuffix { get; set; } public bool hasTrailingCurrency { get; set; } public double PriceDoubleGet() { var fmt = new NumberFormatInfo(); fmt.NumberDecimalSeparator = separator; double r; if (double.TryParse(string.Format("{0}{1}{2}", integer, separator, decimals), NumberStyles.Any, fmt, out r)) { return r; } return 0; } } public class Priceformat { public string currencyPrefix { get; set; } public string currencySuffix { get; set; } public string priceFormatter { get; set; } } } //namespace Availability //{ // public class AvailabilityResponse // { // public Stockavailability StockAvailability { get; set; } // } // public class Stockavailability // { // public Classunitkey ClassUnitKey { get; set; } // public Itemkey ItemKey { get; set; } // public Retailitemavailability RetailItemAvailability { get; set; } // public Availablestockforecastlist AvailableStockForecastList { get; set; } // public Xmlns xmlns { get; set; } // } // public class Classunitkey // { // public Classtype ClassType { get; set; } // public Classunittype ClassUnitType { get; set; } // public Classunitcode ClassUnitCode { get; set; } // } // public class Classtype // { // [JsonProperty("$")] // public string ItemVal { get; set; } // } // public class Classunittype // { // [JsonProperty("$")] // public string ItemVal { get; set; } // } // public class Classunitcode // { // [JsonProperty("$")] // public string ItemVal { get; set; } // } // public class Itemkey // { // public Itemno ItemNo { get; set; } // public Itemtype ItemType { get; set; } // } // public class Itemno // { // [JsonProperty("$")] // public string ItemVal { get; set; } // } // public class Itemtype // { // [JsonProperty("$")] // public string ItemVal { get; set; } // } // public class Retailitemavailability // { // public Availablestock AvailableStock { get; set; } // public Availablestocktype AvailableStockType { get; set; } // public Instockprobabilitycode InStockProbabilityCode { get; set; } // public Instockrangecode InStockRangeCode { get; set; } // public Incustomerorderrangecode InCustomerOrderRangeCode { get; set; } // public Retailitemcommchildavailabilitylist RetailItemCommChildAvailabilityList { get; set; } // } // public class Availablestock // { // [JsonProperty("$")] // public int? ItemVal { get; set; } // } // public class Availablestocktype // { // [JsonProperty("$")] // public string ItemVal { get; set; } // } // public class Instockprobabilitycode // { // [JsonProperty("$")] // public string ItemVal { get; set; } // } // public class Instockrangecode // { // [JsonProperty("$")] // public string ItemVal { get; set; } // } // public class Incustomerorderrangecode // { // [JsonProperty("$")] // public string ItemVal { get; set; } // } // public class Retailitemcommchildavailabilitylist // { // public Retailitemcommchildavailability[] RetailItemCommChildAvailability { get; set; } // } // public class Retailitemcommchildavailability // { // //public Itemno1 ItemNo { get; set; } // public Itemno ItemNo { get; set; } // //public Itemtype1 ItemType { get; set; } // public Itemtype ItemType { get; set; } // //public Availablestocktype1 AvailableStockType { get; set; } // public Availablestocktype AvailableStockType { get; set; } // public Recommendedsaleslocation RecommendedSalesLocation { get; set; } // public Salesmethodcode SalesMethodCode { get; set; } // } // //public class Itemno1 // //{ // // public object _ { get; set; } // //} // //public class Itemtype1 // //{ // // public string _ { get; set; } // //} // //public class Availablestocktype1 // //{ // // public string _ { get; set; } // //} // public class Recommendedsaleslocation // { // } // public class Salesmethodcode // { // [JsonProperty("$")] // public string ItemVal { get; set; } // } // public class Availablestockforecastlist // { // public Availablestockforecast[] AvailableStockForecast { get; set; } // } // public class Availablestockforecast // { // //public Availablestock1 AvailableStock { get; set; } // public Availablestock AvailableStock { get; set; } // //public Availablestocktype2 AvailableStockType { get; set; } // public Availablestocktype AvailableStockType { get; set; } // //public Instockprobabilitycode1 InStockProbabilityCode { get; set; } // public Instockprobabilitycode InStockProbabilityCode { get; set; } // public Validdatetime ValidDateTime { get; set; } // public Validdatetimeunit ValidDateTimeUnit { get; set; } // } // //public class Availablestock1 // //{ // // public int _ { get; set; } // //} // //public class Availablestocktype2 // //{ // // public string _ { get; set; } // //} // //public class Instockprobabilitycode1 // //{ // // public string _ { get; set; } // //} // public class Validdatetime // { // [JsonProperty("$")] // public string ItemVal { get; set; } // } // public class Validdatetimeunit // { // [JsonProperty("$")] // public string ItemVal { get; set; } // } // public class Xmlns // { // [JsonProperty("$")] // public string ItemVal { get; set; } // } //} namespace Availability { public class AvailabilityObject { public Availability[] availabilities { get; set; } public SalesLocation[] salesLocations { get; set; } public AvailabilityDataItem[] data { get; set; } public DateTime timestamp { get; set; } public string traceId { get; set; } } public class AvailabilityDataItem { public bool? isInCashAndCarryRange { get; set; } public bool isInHomeDeliveryRange { get; set; } public AvailableStock[] availableStocks { get; set; } public Classunitkey classUnitKey { get; set; } public Itemkey itemKey { get; set; } } public class AvailableStock { [JsonProperty("type")] public string _type { get; set; } public int quantity { get; set; } } public class SalesLocation { public Childitem[] childItems { get; set; } public Classunitkey classUnitKey { get; set; } public Itemkey itemKey { get; set; } public Saleslocation[] salesLocations { get; set; } } //public class Classunitkey //{ // public string classUnitCode { get; set; } // public string classUnitType { get; set; } //} //public class Itemkey //{ // public string itemNo { get; set; } // public string itemType { get; set; } //} public class Childitem { public Itemkey itemKey { get; set; } public Saleslocation[] salesLocations { get; set; } } //public class Itemkey1 //{ // public string itemNo { get; set; } // public string itemType { get; set; } //} public class Saleslocation { public Aisleandbin aisleAndBin { get; set; } public string division { get; set; } public string id { get; set; } public int recommendationRank { get; set; } public string type { get; set; } } public class Aisleandbin { public string aisle { get; set; } public string bin { get; set; } } public class Availability { public bool availableForCashCarry { get; set; } public bool availableForClickCollect { get; set; } public bool availableForHomeDelivery { get; set; } public Buyingoption buyingOption { get; set; } public Classunitkey classUnitKey { get; set; } public Itemkey itemKey { get; set; } } public class Buyingoption { public Cashcarry cashCarry { get; set; } public Homedelivery homeDelivery { get; set; } } public class Cashcarry { public Availability1 availability { get; set; } public Range range { get; set; } public string unitOfMeasure { get; set; } } public class Availability1 { public Probability probability { get; set; } public int quantity { get; set; } public DateTime updateDateTime { get; set; } public Restock[] restocks { get; set; } } public class Probability { public Thisday thisDay { get; set; } public DateTime updateDateTime { get; set; } } public class Thisday { public Colour colour { get; set; } public string messageType { get; set; } } public class Colour { public string rgbDec { get; set; } public string rgbHex { get; set; } } public class Restock { public string earliestDate { get; set; } public string latestDate { get; set; } public int quantity { get; set; } public string reliability { get; set; } public string type { get; set; } public DateTime updateDateTime { get; set; } } public class Range { public bool inRange { get; set; } } public class Homedelivery { public Range1 range { get; set; } public Availability2 availability { get; set; } } public class Range1 { public bool inRange { get; set; } } public class Availability2 { public Probability1 probability { get; set; } public DateTime updateDateTime { get; set; } } public class Probability1 { public Thisday1 thisDay { get; set; } public DateTime updateDateTime { get; set; } } public class Thisday1 { public Colour1 colour { get; set; } public string messageType { get; set; } } public class Colour1 { public string rgbDec { get; set; } public string rgbHex { get; set; } } public class Classunitkey { public string classUnitCode { get; set; } public string classUnitType { get; set; } } public class Itemkey { public string itemNo { get; set; } public string itemType { get; set; } } } namespace Cataloge { public class Category { public string id { get; set; } public int depth { get; set; } public List<Subcategory> subcategories { get; set; } public Category() { subcategories = new List<Subcategory>(); } } public class Subcategory { public string id { get; set; } public string link { get; set; } public string title { get; set; } public bool isCrossReferenced { get; set; } } } namespace Media { public class MediaListBase { public string type { get; set; } public Content content { get; set; } } public class MediaGrid { public List<Fullmedialist> fullMediaList { get; set; } public List<Medialist> mediaList { get; set; } public int heroesCount { get; set; } public bool isDoubleHero { get; set; } public bool isSingleHero { get; set; } public bool disabledButton { get; set; } public string buttonMoreLabel { get; set; } public string buttonLessLabel { get; set; } public int barPosition { get; set; } public float barWidth { get; set; } public bool isSingleMedia { get; set; } public object energyLabelProps { get; set; } public MediaGrid() { fullMediaList = new List<Fullmedialist>(); mediaList = new List<Medialist>(); } } public class Fullmedialist : MediaListBase { //public string type { get; set; } //public Content content { get; set; } } public class Content { public string alt { get; set; } public string id { get; set; } public string imageFileName { get; set; } public string url { get; set; } public string type { get; set; } } public class Medialist : MediaListBase { //public string type { get; set; } //public Content content { get; set; } } } namespace Pagination { public class DataCategory { public string id { get; set; } public Pagination pagination { get; set; } public int totalCount { get; set; } } public class Pagination { public int currentPage { get; set; } public Next next { get; set; } public object prev { get; set; } } public class Next { public string url { get; set; } public string label { get; set; } } //public class MoreproductsResponse //{ // public string usergroup { get; set; } // public Moreproducts moreProducts { get; set; } //} //public class Moreproducts //{ // public List<Productwindow> productWindow { get; set; } // public Moreproducts() // { // productWindow = new List<Productwindow>(); // } //} //public class Productwindow //{ // public string name { get; set; } // public string typeName { get; set; } // public string itemMeasureReferenceText { get; set; } // public string mainImageUrl { get; set; } // public string pipUrl { get; set; } // public string id { get; set; } // public string metadata { get; set; } // public bool onlineSellable { get; set; } // public Gprdescription gprDescription { get; set; } // public Color[] colors { get; set; } // public float priceNumeral { get; set; } // public string currencyCode { get; set; } //} //public class Gprdescription //{ // public object[] colors { get; set; } // public int numberOfSizes { get; set; } // public int numberOfColors { get; set; } //} //public class Color //{ // public string name { get; set; } // public string id { get; set; } //} public class MoreproductsResponse { public string usergroup { get; set; } public Moreproducts moreProducts { get; set; } } public class Moreproducts { public List<Productwindow> productWindow { get; set; } public Moreproducts() { productWindow = new List<Productwindow>(); } } public class Productwindow { public string name { get; set; } public string typeName { get; set; } public string itemMeasureReferenceText { get; set; } public string mainImageUrl { get; set; } public string pipUrl { get; set; } public string id { get; set; } public string itemNoGlobal { get; set; } public string metadata { get; set; } public bool onlineSellable { get; set; } public bool lastChance { get; set; } public Gprdescription gprDescription { get; set; } public Color[] colors { get; set; } public double priceNumeral { get; set; } public string currencyCode { get; set; } } public class Gprdescription { public int numberOfVariants { get; set; } public List<Variant> variants { get; set; } public Gprdescription() { variants = new List<Variant>(); } } public class Variant { public string id { get; set; } public string pipUrl { get; set; } public string imageUrl { get; set; } public string imageAlt { get; set; } //public Price price { get; set; } } public class Price { public string prefix { get; set; } public string wholeNumber { get; set; } public string separator { get; set; } public string decimals { get; set; } public string suffix { get; set; } public bool isRegularCurrency { get; set; } } public class Color { public string name { get; set; } public string id { get; set; } public string hex { get; set; } } } namespace ProductProps { /* * akl_02152021: * Теперь информация об упаковках приходит внутри html страницы товара. */ public class GeneralDataProps { public string productName { get; set; } public bool hideSection { get; set; } public string productNumber { get; set; } public string productType { get; set; } public bool onlineSellable { get; set; } public bool enabledCartButton { get; set; } public string noJavascriptLabel { get; set; } public string currency { get; set; } public string productPrice { get; set; } public string ratings { get; set; } } public class Props { public bool isClient { get; set; } public JToken dimensionProps { get; set; } public Dimensionprops DimensionPropsObject { get { Dimensionprops dp = null; try { var jo = dimensionProps as JObject; if (jo != null) { dp = jo.ToObject<Dimensionprops>(); } } catch { } return dp; } } public Productdetailsprops productDetailsProps { get; set; } // public object productReviewsProps { get; set; } //public bool technicalInformationProps { get; set; } } public class Dimensionprops { public string title { get; set; } public Packaging packaging { get; set; } //public Dimension[] dimensions { get; set; } //public object[] images { get; set; } //public Fallbackimage fallbackImage { get; set; } //public string eventLabel { get; set; } } public class Fallbackimage { public string alt { get; set; } public string id { get; set; } public string imageFileName { get; set; } public string url { get; set; } public string type { get; set; } } public class Dimension { public string type { get; set; } public string name { get; set; } public string measure { get; set; } } public class Productdetailsprops { public string title { get; set; } public string eventLabel { get; set; } public Productdescriptionprops productDescriptionProps { get; set; } public Accordionobject accordionObject { get; set; } } public class Productdescriptionprops { public string[] paragraphs { get; set; } public string designerLabel { get; set; } public string designerName { get; set; } } public class Accordionobject { public Materialsandcare materialsAndCare { get; set; } public object sustainabilityAndEnvironment { get; set; } public Packaging packaging { get; set; } public Assemblyanddocuments assemblyAndDocuments { get; set; } } public class Materialsandcare { public string id { get; set; } public string title { get; set; } public Contentprops contentProps { get; set; } } public class Contentprops { public Material[] materials { get; set; } public object[] careInstructions { get; set; } public object[] proposition65 { get; set; } } public class Material { public string productType { get; set; } public Material1[] materials { get; set; } } public class Material1 { public string part { get; set; } public string material { get; set; } } public class Packaging { public string id { get; set; } public string title { get; set; } public Contentprops1 contentProps { get; set; } } public class Contentprops1 { public List<Package> packages { get; set; } public string totalNoOfPackagesText { get; set; } public Contentprops1() { packages = new List<Package>(); } } public class Package { public string name { get; set; } public string typeName { get; set; } public Articlenumber articleNumber { get; set; } public Measurement[][] measurements { get; set; } public Quantity quantity { get; set; } public string multiPackDisclaimerText { get; set; } } public class Articlenumber { public string label { get; set; } public string value { get; set; } } public class Quantity { public string label { get; set; } public int value { get; set; } } public class Measurement { public string label { get; set; } public string value { get; set; } } public class Assemblyanddocuments { public string id { get; set; } public string title { get; set; } public Contentprops2 contentProps { get; set; } } public class Contentprops2 { public Attachments attachments { get; set; } } public class Attachments { public Assembly assembly { get; set; } public Software software { get; set; } public Other other { get; set; } public Disassembly disassembly { get; set; } } public class Assembly { public string header { get; set; } public Attachment[] attachments { get; set; } } public class Attachment { public string id { get; set; } public string url { get; set; } public string label { get; set; } } public class Software { public string header { get; set; } public object[] attachments { get; set; } } public class Other { public string header { get; set; } public object[] attachments { get; set; } } public class Disassembly { public string header { get; set; } public object[] attachments { get; set; } } } } }