Ikea.com country universal scraper

Here is a sample of release code for ikea web scraper.
This code developed using our internal library for c# but you can adjust it for your case without using our internal dll.
Just replace data loading function with standart .net webclient or web request.

[code lang=”csharp”]
using System;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using System.Globalization;
using System.Web;
using CatalogLoaderCommon;
using HtmlAgilityPack;
using System.Linq;
using HtmlDocument = HtmlAgilityPack.HtmlDocument;

using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System.Net;
using RestSharp.Extensions;

#if NET462
using Microsoft.CSharp; // for dynamics
#endif

namespace CatalogLoader._MyWebScrapers
{
public class ikea_scraper : CustomScriptBase
{
public HtmlPageLoader HplCatalog { get; set; }
private Product _product;
private Category _category;
private TaskInfo _mti;

public ikea_scraper() { }

internal enum Region
{
EN, PL, RU
}

internal sealed class IkeaSettings
{
public RegionSettings RegionSettings { get; set; }
public Region Region { get; set; }
public List<string> StoreIDs { get; set; }
public int ProductQtyMin { get; set; }
public double ProductPriceMin { get; set; }
public bool CatalogModeEnabled { get; set; }
public bool PackagesWeightTotalAsAttribute { get; set; }
public bool ProductSizesAsPackagesSizesTotal { get; set; }
}
internal sealed class RegionSettings
{
public string Domain { get; set; }
public string RootCategoryUrl { get; set; }
public string ShopAllDetectorString { get; set; }
public string ProductAvailabilityUrlTemplate { get; set; }
public string MoreProductsUrl { get; set; }
}

public override void GrabCatalogBuild(GrabCatalogBuildScriptParameters p)
{
//Init(p);
_category = new Category() { ID = "0", SourceUrl = _ikeaSettings.RegionSettings.RootCategoryUrl };
var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty();
hpl.Load(_category.SourceUrl);
CatalogBuild(hpl.HtmlDoc.DocumentNode, _category);
foreach (var bottom in _category.CategoriesBottomGet())
{
CatalogBuildBottoms(bottom);
}
//foreach (var c in _category.GetCategoryList())
//{
// c.SourceUrl = CategoryUrlWithIdCompose(c);
//}
p.Root = _category;
}

private void CatalogBuildBottoms(Category parent)
{
if (!_mti.CanContinue())
{
return;
}

_mti.AddLogFormatted("!!! CatalogBuildBottoms() started. Parent => {0}.", parent);
var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty();
if (hpl.Load(parent.SourceUrl))
{
var subLinks = hpl.HtmlDoc.DocumentNode.SelectNodes(".//div[@class=’plp-page-container__main’]//div[@class=’plp-navigation-slot-wrapper’]//nav[@role=’navigation’][@data-visual-navigation-links]/a");
if (subLinks != null)
{
foreach (var link in subLinks)
{
string name = link.InnerText;
string href = link.GetAttributeValue("href", "");
string data_tracking_label = link.GetAttributeValue("data-tracking-label", "");
if (string.IsNullOrEmpty(name) || string.IsNullOrEmpty(href) || AlreadyStoredForParent(parent, href))
{
continue;
}
var sub = new Category() { Name = HttpUtility.HtmlDecode(name.Trim()), SourceUrl = href };
string id = CategoryIdParse(data_tracking_label);
if (!string.IsNullOrEmpty(id))
{
sub.ID = id;
}
parent.AddCategory(sub);
CatalogBuildBottoms(sub);
}
}
}
}

private bool AlreadyStoredForParent(Category parent, string possibleNewHref)
{
return parent.GetParents(true).Any(c => c.SourceUrl == possibleNewHref) || parent.GetCategoryList(true).Any(c => c.SourceUrl == possibleNewHref);
}

private void CatalogBuild(HtmlNode node, Category parent)
{
if (node == null)
{
return;
}
var subNodes = node.SelectNodes("./li");
if (subNodes != null)
{
foreach (var subNode in subNodes)
{
var link = subNode.SelectSingleNode("./a");
if (link != null)
{
var href = link.GetAttributeValue("href", "");
var data_tracking_label = link.GetAttributeValue("data-tracking-label", "");

if (string.IsNullOrEmpty(href)
|| !string.IsNullOrEmpty(data_tracking_label) && data_tracking_label.Equals("all", StringComparison.InvariantCultureIgnoreCase)
|| link.InnerText.IndexOf(_ikeaSettings.RegionSettings.ShopAllDetectorString, StringComparison.InvariantCultureIgnoreCase) >= 0)
{
continue;
}

private string CategoryUrlWithIdCompose(Category cat, string id = null)
{
if (string.IsNullOrEmpty(id))
{
id = cat.ID;
}
return string.Format("{0}@@{1}", cat.SourceUrl, id);
}

private string CategoryIdParse(string data_tracking_label)
{
if (!string.IsNullOrEmpty(data_tracking_label))
{
var arr = data_tracking_label.Split(new[] { "|" }, StringSplitOptions.RemoveEmptyEntries);
if (arr.Length > 0)
{
return arr[arr.Length – 1].Trim();
}
}
return "";
}

private GrabProcessState _gps;
private CacheCustom _cacheCustom = null;
private IkeaSettings _ikeaSettings;
public override void Login(LoginScriptParameters p)
{
_gps = p.State as GrabProcessState;
_mti = p.Process.m_ti;
p.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36";

_ikeaSettings = IkeaSettingsBuild(_gps.GrabberSettings);
_ikeaSettings.RegionSettings = RegionSettingsBuild(_ikeaSettings.Region);

_gps.GrabberSettings.Settings.CacheEnabled = true; // Make enabled by default.
_cacheCustom = new CacheCustom(_gps.CacheGet().m_cacheStorage);
}

private IkeaSettings IkeaSettingsBuild(GrabberSettings gs)
{
var ikeaSettings = new IkeaSettings();

string minQty = gs.UserParameterGet("ProductQtyMin");
if (!string.IsNullOrEmpty(minQty))
{
int mq = 0;
if (int.TryParse(minQty, out mq))
{
ikeaSettings.ProductQtyMin = mq;
}
}

string minPrice = gs.UserParameterGet("ProductPriceMin");
if (!string.IsNullOrEmpty(minPrice))
{
double mp = 0;
if (double.TryParse(minPrice, out mp))
{
ikeaSettings.ProductPriceMin = mp;
}
}

string regionName = gs.UserParameterGet("Region");
if (string.IsNullOrEmpty(regionName))
{
regionName = "EN";
}

Region r;
if (Enum.TryParse(regionName.ToUpper(), out r))
{
ikeaSettings.Region = r;
}

ikeaSettings.CatalogModeEnabled = true;
string catalogModeEnabled = gs.UserParameterGet("CatalogModeEnabled");
if (!string.IsNullOrEmpty(catalogModeEnabled))
{
bool b;
if (bool.TryParse(catalogModeEnabled, out b))
{
ikeaSettings.CatalogModeEnabled = b;
}
}

string pckgsWeightTotalAsAttr = gs.UserParameterGet("PackagesWeightTotalAsAttribute");
if (!string.IsNullOrEmpty(pckgsWeightTotalAsAttr))
{
bool b;
if (bool.TryParse(pckgsWeightTotalAsAttr, out b))
{
ikeaSettings.PackagesWeightTotalAsAttribute = b;
}
}

string prodSizesAsPckgsSizesTotal = gs.UserParameterGet("ProductSizesAsPackagesSizesTotal");
if (!string.IsNullOrEmpty(prodSizesAsPckgsSizesTotal))
{
bool b;
if (bool.TryParse(prodSizesAsPckgsSizesTotal, out b))
{
ikeaSettings.ProductSizesAsPackagesSizesTotal = b;
}
}

string storeIds = gs.UserParameterGet("StoreId");
if (!string.IsNullOrEmpty(storeIds))
{
ikeaSettings.StoreIDs = storeIds.Split(new[] { "," }, StringSplitOptions.RemoveEmptyEntries).ToList();
}

return ikeaSettings;
}

private RegionSettings RegionSettingsBuild(Region r)
{
var rs = new RegionSettings();
switch (r)
{
case Region.EN:
rs.Domain = "https://www.ikea.com/us/en/";
rs.RootCategoryUrl = "https://www.ikea.com/us/en/header-footer/menu-products.html";
rs.ShopAllDetectorString = "Shop all";
rs.ProductAvailabilityUrlTemplate = "https://api.ingka.ikea.com/cia/availabilities/ru/us?itemNos={0}&expand=StoresList,Restocks";
rs.MoreProductsUrl = "https://sik.search.blue.cdtapps.com/us/en/product-list-page/more-products?category=&sort=RELEVANCE&start=&end=&c=lf&v=20200617";
break;
case Region.PL:
rs.Domain = "https://www.ikea.com/pl/pl/";
rs.RootCategoryUrl = "https://www.ikea.com/pl/pl/header-footer/menu-products.html";
rs.ShopAllDetectorString = "Zobacz wszystko";
rs.ProductAvailabilityUrlTemplate = "https://api.ingka.ikea.com/cia/availabilities/ru/pl?itemNos={0}&expand=StoresList,Restocks";
rs.MoreProductsUrl = "https://sik.search.blue.cdtapps.com/pl/pl/product-list-page/more-products?category=&sort=RELEVANCE&start=&end=&c=lf&v=20200617";
break;
case Region.RU:
rs.Domain = "https://www.ikea.com/ru/ru/";
rs.RootCategoryUrl = "https://www.ikea.com/ru/ru/header-footer/menu-products.html";
rs.ShopAllDetectorString = "Смотреть все";
rs.ProductAvailabilityUrlTemplate = "https://api.ingka.ikea.com/cia/availabilities/ru/ru?itemNos={0}&expand=StoresList,Restocks";
rs.MoreProductsUrl = "https://sik.search.blue.cdtapps.com/ru/ru/product-list-page/more-products?category=&sort=RELEVANCE&start=&end=&c=lf&v=20200617";
break;
default:
break;
}
return rs;
}

private Dictionary<string, Dictionary<string, string>> _mainProductUrl2VariantsIds2Links = new Dictionary<string, Dictionary<string, string>>();

public override void GetProductLinksForCategory(CatalogLoaderCommon.GetProductLinksForCategoryScriptParameters p)
{
var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty();
if (!hpl.Load(p.Category.SourceUrl))
{
return;
}

_mti.AddLogInfo($"_ikeaSettings.CatalogModeEnabled ? {_ikeaSettings.CatalogModeEnabled}");

if (_ikeaSettings.CatalogModeEnabled)
{
Paginator paginator = new Paginator(_ikeaSettings.RegionSettings);
string msg;
if (!paginator.Init(hpl.HtmlDoc, out msg))
{
_mti.AddLogError(string.Format("Paginator init failed. Message:'{0}’.", msg));
return;
}
int newLinksCnt = 0;
while (paginator.MoveNext())
{
string nextPage = paginator.MoreProductsLinkBuild();
List<string> links = new List<string>();
if (hpl.Load(nextPage))
{
try
{
var resp = JsonConvert.DeserializeObject<ModelsIkea.Pagination.MoreproductsResponse>(hpl.Content);
//links = resp.moreProducts.productWindow.Select(pw => pw.pipUrl);
foreach (var pw in resp.moreProducts.productWindow)
{
links.Add(pw.pipUrl);

/*
* https://www.ikea.com/ru/ru/cat/vazy-dlya-cvetov-10776/
* http://storage.mydataprovider.com/screenshots/akl/2021-02-11_14-47-22_FLmTFPxU.png
*
* Не у всех товаров на их странице есть варианты,
* хотя на странице категории варианты у товара могут быть.
* В MoreproductsResponse теперь приходит информация о вариантах.
* Поэтому сохраняем варианты для ссылок на основной товар, чтобы взять
* их при отсутствии вариантов на странице товара.
*
*/
if (pw.gprDescription != null && pw.gprDescription.variants != null)
{
_mainProductUrl2VariantsIds2Links[pw.pipUrl] = pw.gprDescription.variants.ToDictionary(v => v.id, v => v.pipUrl);
}
}
}
catch (Exception ex)
{
_mti.AddLogError(ex.ToString());
}
}
else
{
break;
}

if (links == null || !links.Any())
{
_mti.AddLogInfo("Can’t find new links.");
break;
}
foreach (var link in links)
{
p.Category.ProductLinks.Add(link);
newLinksCnt++;
if (newLinksCnt == hpl.Settings.MaxProductsInCategory)
{
goto OUT_OF_PAGIN;
}
}
}
OUT_OF_PAGIN:
_mti.AddLogFormatted("GetProductLinksForCategory finished. Links count:{0}.", newLinksCnt.ToString());
}
else
{
_gps.CacheGet().OnPageLoadedWithError += Cache_OnPageLoadedWithError;

// В этих настройках парсим ссылки на все товары из категорий.
// https://www.ikea.com/ru/ru/header-footer/menu-products.html
// http://storage.mydataprovider.com/screenshots/akl/2020-10-23_09-11-43_XHHft6PA.png
// Если товаров в ней нет, идем в подкатегории.
// ==================================================================================
//
// akl_03122020
// !!! UPDATE:
//
// Не во всех категориях "Смотреть все товары" возвращаются реально все товары.
// Напр., "Товары для ремонта" https://www.ikea.com/ru/ru/cat/tovary-dlya-remonta-hi001/
// не содержит все товары
// из "Шурупы и крепления" https://www.ikea.com/ru/ru/cat/shurupy-i-krepleniya-49768/
// Поэтому, начинаем обход с подкатегорий.

try
{
TestCategory testCategory = null;
/*
testCategory = new TestCategory()
{
Root = "Кухни и бытовая техника",
Sub = "Кухни",
BottomId = "ka003"
};
*/
if (!string.IsNullOrEmpty(_gps.GrabberSettings.UserParameterGet("TEST_CATEGORY")))
{
try
{
testCategory = JsonConvert.DeserializeObject<TestCategory>(_gps.GrabberSettings.UserParameterGet("TEST_CATEGORY"));
}
catch { }
}
AllProductsProcess(p, testCategory: testCategory);
}
catch (Exception ex)
{
_mti.AddLogError(ex);
}
finally
{
_gps.CacheGet().OnPageLoadedWithError -= Cache_OnPageLoadedWithError;
//_productsPrcsdLinks.Clear();
}
_mti.AddLogFormatted("GetProductLinksForCategory finished. Links count:{0}.", _productsPrcsdLinks.Count.ToString());
}
}

public class TestCategory
{
private string _rootXpath;
private string _subXpath;
public string Root { get; set; }
public string Sub { get; set; }
internal string RootXpath
{
get
{
if (string.IsNullOrEmpty(_rootXpath))
{
_rootXpath = string.Format("./li/a[contains(.,'{0}’)]", Root);
}
return _rootXpath;
}
}
internal string SubXpath
{
get
{
if (string.IsNullOrEmpty(_subXpath))
{
_subXpath = string.Format("./nav/ul/li/a[contains(.,'{0}’)]", Sub);
}
return _subXpath;
}
}
public string BottomId { get; set; }
}

private List<string> _productsPrcsdLinks = new List<string>(100000);
private Dictionary<string, string> _categoryName2Url = new Dictionary<string, string>();
private Dictionary<string, string> _productUrl2FullCategoryName = new Dictionary<string, string>();
private void AllProductsProcess(GetProductLinksForCategoryScriptParameters p, List<string> allProductsLinks = null, TestCategory testCategory = null)
{
int maxProductsInCategory = 100000;
var allProductsLink2AllProductsLinkNode = new Dictionary<string, HtmlNode>();
var allProductsLink2CategoryName = new Dictionary<string, string>();

var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty();
if (allProductsLinks == null)
{
allProductsLinks = new List<string>();

hpl.Load(p.Category.SourceUrl);
string categoriesLinksXPath = testCategory != null ? testCategory.RootXpath : "./li/a";
var categoriesLinks = hpl.HtmlDoc.DocumentNode.SelectNodes(categoriesLinksXPath);
if (categoriesLinks == null)
{
_mti.AddLogWarning("!!! Can’t parse categories links.");
return;
}

foreach (var catLink in categoriesLinks)
{
if (!_mti.CanContinue() || string.IsNullOrEmpty(catLink.InnerText))
{
continue;
}

var categoryName = catLink.InnerText.Trim();
_categoryName2Url[categoryName] = TextUtils.UrlNewCompose(_ikeaSettings.RegionSettings.Domain, catLink.Attributes["href"].Value);

string subCategoriesLinksXPath = testCategory != null ? testCategory.SubXpath : $"./nav/ul/li/a[not(contains(.,'{_ikeaSettings.RegionSettings.ShopAllDetectorString}’))]";
var subCategoriesLinksNodes = catLink.ParentNode.SelectNodes(subCategoriesLinksXPath);
if (subCategoriesLinksNodes == null)
{
_mti.AddLogWarning(string.Format("!!! Category:'{0}’ => can’t find subcategories link.", categoryName));
continue;
}

_mti.AddLogWarning(string.Format("!!! Category:'{0}’ => subcategories links count:{1}.", categoryName, subCategoriesLinksNodes.Count.ToString()));

foreach (var subCategoryNode in subCategoriesLinksNodes)
{
if (!_mti.CanContinue())
{
continue;
}
string subCategoryLink = subCategoryNode.GetAttributeValue("href", "");
if (!string.IsNullOrEmpty(subCategoryLink))
{
string subCategoryName = subCategoryNode.InnerText.Trim();
subCategoryLink = TextUtils.UrlNewCompose(_ikeaSettings.RegionSettings.Domain, subCategoryLink);
_categoryName2Url[subCategoryName] = subCategoryLink;
string categoryId = CategoryIdParseFromUrl(subCategoryLink);

if (testCategory != null)
{
categoryId = testCategory.BottomId;
}

_mti.AddLogInfo(string.Format("!!! Subcategory link:'{0}’ => category ID:'{1}’.", subCategoryLink, categoryId));
if (string.IsNullOrEmpty(categoryId))
{
continue;
}

/*
* akl_03022021:
* Т.к. утоваров без крошек в карточке отсутствует ссылка на их непосредственную категорию,
* придется сделать минимум еще один запрос, чтобы собрать имена-ссылки подкатегорий здесь.
*/
if (hpl.Load(subCategoryLink))
{
foreach (var name2url in CategoriesName2UrlParse(hpl.Content))
{
_categoryName2Url[name2url.Key] = name2url.Value;
}
}

var links = new List<string>();
Models.Pagination.MoreproductsResponse resp = null;

if (hpl.Load(string.Format("https://sik.search.blue.cdtapps.com/ru/ru/product-list-page/more-products?category={0}&sort=RELEVANCE&start=0&end={1}&c=lf&v=20210126",
categoryId, maxProductsInCategory.ToString())))
{
try
{
resp = JsonConvert.DeserializeObject<Models.Pagination.MoreproductsResponse>(hpl.Content);
//links = resp.moreProducts.productWindow.Select(pw => pw.pipUrl);
foreach (var pw in resp.moreProducts.productWindow)
{
links.Add(pw.pipUrl);

/*
* akl_02262021:
* Есть товары, в карточке которых отсутствуют крошки. По этой причине,
* нужно мапить ссылку товара на полную категорию на этапе сбора ссылок.
*/
_productUrl2FullCategoryName[pw.pipUrl] = string.Format("{0}->{1}", categoryName, subCategoryName);
}
}
catch (Exception ex)
{
_mti.AddLogError(ex.ToString());
}
}
else
{
continue;
}

_mti.AddLogFormatted("!!! Category Link ‘{0}’ => products links num by api: {1}.", subCategoryLink, links != null ? links.Count().ToString() : "0");

//if (links == null)
//{
// continue;
//}
foreach (var link in links)
{
if (_productsPrcsdLinks.Contains(link))
{
_mti.AddLogFormatted("!!! Product link ‘{0}’ stored allready.", link);
continue;
}

_productsPrcsdLinks.Add(link);

p.Category.ProductLinks.Add(link);
if (_productsPrcsdLinks.Count == hpl.Settings.MaxProductsInCategory)
{
goto OUT_OFF_LOOPS;
}
}
}
}
}
}

OUT_OFF_LOOPS:
_mti.AddLogInfo("[AllProductsProcess] Finished.");
}

private string CategoryIdParseFromUrl(string link)
{
var urlNormal = TextUtils.UrlNewCompose(_ikeaSettings.RegionSettings.Domain, link);
Uri u;
if (Uri.TryCreate(urlNormal, UriKind.RelativeOrAbsolute, out u))
{
return Regex.Replace(u.AbsolutePath, @"^[\w\W]+-", "").TrimEnd(‘/’);
}
return "";
}

private List<string> _bottomsCategoriesXpathList = new List<string>()
{
".//div[contains(@class,’plp-category-link-list’)]//li[contains(@class,’plp-category-link-list__item’)]/a",
".//nav[@role=’navigation’]/a"
};
private Dictionary<string, string> CategoriesName2UrlParse(string html)
{
var hDoc = new HtmlDocument();
hDoc.LoadHtml(html);
foreach (var xpath in _bottomsCategoriesXpathList)
{
var nodes = hDoc.DocumentNode.SelectNodes(xpath);
if (nodes != null)
{
var n2u = new Dictionary<string, string>();
foreach (var n in nodes)
{
string h = n.GetAttributeValue("href", "");
if (string.IsNullOrEmpty(n.InnerText) || string.IsNullOrEmpty(h))
{
continue;
}
n2u[n.InnerText.Trim()] = TextUtils.UrlNewCompose(_ikeaSettings.RegionSettings.Domain, h);
}
_mti.AddLogInfo("[CategoriesName2UrlParse] Result ==================");
_mti.AddLogInfo(string.Join("\r\n", n2u.Select(kvp => string.Format("{0} => {1}", kvp.Key, kvp.Value))));
_mti.AddLogInfo("====================================================");

return n2u;
}
}
return new Dictionary<string, string>();
}

private bool Cache_OnPageLoadedWithError(Cache cacheProvider, HttpWebResponse response, out string content)
{
content = "";

if (response.ContentLength != 0)
{
_mti.AddLogWarning("In CustomScript_OnPageLoadedWithError");
try
{
var dataContent = response.GetResponseStream().ReadAsBytes();
var exceptionContent = cacheProvider.Encoding.GetString(dataContent);
if (exceptionContent.Contains("Unknown category key"))
{
content = exceptionContent;
return true;
}
}
catch (Exception)
{
}
}

return !string.IsNullOrEmpty(content);
}

internal class Paginator
{
/*
* https://sik.search.blue.cdtapps.com/ru/ru/product-list-page/more-products?sessionId=5980bee3-2658-4788-bb3c-501cd50287f3&category=11703&sort=RELEVANCE&start=24&end=48&c=lf&v=20200617
* Значение параметра запроса "sessionId" совпадает с local storage ключем "search-sik-session-ruru".
* Источник возвращает валидные данные и без "sessionId", поэтому, пока что, игнорим этот параметр.
*/
private int _currentIdx;
private int _lastIdx = 0;
private int _shift;
private ModelsIkea.Pagination.DataCategory _dataCategory;
private RegionSettings _regionSettings;
public Paginator(RegionSettings rs, int startIdx = 0, int shift = 24)
{
_regionSettings = rs;
_currentIdx = startIdx – 1;
_shift = shift;
}
internal bool Init(HtmlDocument htmlDoc, out string msg)
{
msg = string.Empty;
bool r = false;
string dataCategoryStr = TextUtils.GetHtmlValue(htmlDoc, ".//div[contains(@class,’js-product-list’)]", "data-category", true);
if (!string.IsNullOrEmpty(dataCategoryStr))
{
try
{
_dataCategory = JsonConvert.DeserializeObject<ModelsIkea.Pagination.DataCategory>(dataCategoryStr);
_lastIdx = _dataCategory.totalCount / _shift;
if (_dataCategory.totalCount % _shift > 0)
{
_lastIdx++;
}
r = true;
}
catch (Exception ex)
{
msg = ex.ToString();
}
}
else
{
msg = "dataCategoryStr is EMPTY";
}
return r;
}
internal string MoreProductsLinkBuild()
{
var uBuilder = new UriBuilder(_regionSettings.MoreProductsUrl);
var query = HttpUtility.ParseQueryString(uBuilder.Query);
query["category"] = _dataCategory.id;
query["start"] = (_currentIdx * _shift).ToString();
query["end"] = (_currentIdx * _shift + _shift).ToString();
uBuilder.Query = query.ToString();
return uBuilder.ToString();
}

internal bool MoveNext()
{
_currentIdx++;
return _currentIdx <= _lastIdx;
}
}

public override void RunCategory(RunCategoryScriptParameters p) { }

private void Init(RunProductScriptParameters p)
{
_mti = p.Process.m_ti;
_product = p.Product;
_category = p.Category;
}

public override void RunProduct(RunProductScriptParameters p)
{
try
{
Init(p);

_category = p.Category;
_product.Name = HttpUtility.HtmlDecode(_product.Name);
_product.Name = _product.GetAttributeValue("SYS_PRODUCT_NAME2") + " " + _product.Name;

if (_cacheCustom.IsProcessed(_product.Art))
{
_mti.AddLogFormatted("!!! Product ART:'{0}’ will be skipped because processed early.", _product.Art);
return;
}

if (_product.PriceIsOk)
_product.Price = _product.Price/*.Replace(".", ",")*/.Replace(" ", string.Empty);
else
_product.Price = "0";

bool mainVariantProcessed = false;

bool variantsHtmlFromProductPage = false;
if (!string.IsNullOrEmpty(_product.GetAttributeValue("VARIATION_SECTION")))
{
variantsHtmlFromProductPage = true;
}
else if (!string.IsNullOrEmpty(_product.GetAttributeValue("PRODUCT_STYLES")))
{
//https://www.ikea.com/ru/ru/p/kolbyorn-shkaf-d-doma-sada-zelenyy-40455908/
//для товаров с комбинациями только по одной опции.
variantsHtmlFromProductPage = true;
}

Dictionary<string, string> variantsIDs2Links = new Dictionary<string, string>();
if (variantsHtmlFromProductPage)
{
//VariantsIDs2LinksPopulate(product.GetAttributeValue("VARIATION_SECTION"), product.GetAttributeValue("PRODUCT_STYLES"),
// variantsIDs2Links, new List<string>() { product.Url });
variantsIDs2Links = VariantsIDs2LinksGet(_product.GetAttributeValue("VARIATION_SECTION"), _product.GetAttributeValue("PRODUCT_STYLES"));
}
else
{
var variantsIDs2LinksStoredFromCategoryPage = new Dictionary<string, string>();
if (_mainProductUrl2VariantsIds2Links.TryGetValue(_product.Url, out variantsIDs2LinksStoredFromCategoryPage))
{
_mti.AddLogFormatted("!!! Variants from category page will be used => product url:'{0}’", _product.Url);
variantsIDs2Links = variantsIDs2LinksStoredFromCategoryPage;
//Нужно принудительно обработать вариант основного товара, если берем варианты из категории.
variantsIDs2Links[_product.Art] = _product.Url;
}
}

if (nonProcessedVariantsIds2Links.Count > 0)
{
var vIds2LinksForBulkLoading = new Dictionary<string, string>();
foreach (var id2Link in nonProcessedVariantsIds2Links)
{
string html;
if (!_cacheCustom.Load(id2Link.Value, out html))
{
vIds2LinksForBulkLoading[id2Link.Key] = id2Link.Value;
}
}
if (vIds2LinksForBulkLoading.Count > 0)
{
VariantsBulkLoad(vIds2LinksForBulkLoading);
}

var grabHelper = _gps.ProcessMain as GrabHelper;
foreach (var pr in productsForSave)
{
if (_cacheCustom.IsProcessed(pr.Art))
{
_mti.AddLogFormatted("!!! Product ART:'{0}’ will be skipped because processed early.", pr.Art);
continue;
}
_cacheCustom.SaveAsProcessed(pr.Art);

if ((_ikeaSettings.ProductQtyMin > 0 && pr.QuantityInt() < _ikeaSettings.ProductQtyMin)
|| (_ikeaSettings.ProductPriceMin > 0 && pr.PriceDoubleUnsafe < _ikeaSettings.ProductPriceMin))
{
continue;
}

private void VariantsBulkLoad(Dictionary<string, string> variantsIDs2Links)
{
_mti.AddLogFormatted("[VariantsBulkLoad] Started. variantsIDs2Links:{0}.", variantsIDs2Links.Count);
var bulkRequests = new List<CatalogLoader.Utils.HtmlBulkRequests.HttpWebRequestEntry>();
foreach (var id2Link in variantsIDs2Links)
{
var r = HttpWebRequest.Create(id2Link.Value) as HttpWebRequest;
r.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36";
r.CookieContainer = _gps.m_cookies_con;
bulkRequests.Add(new CatalogLoader.Utils.HtmlBulkRequests.HttpWebRequestEntry()
{
Key = id2Link.Key,
Request = r
});
}

var hbr = HtmlBulkRequestsGet();
_mti.AddLogFormatted("[VariantsBulkLoad] !!! hbr.Process() started => hbr._requests.Count:{0}.", bulkRequests.Count.ToString());
hbr.Process(bulkRequests);
_mti.AddLogFormatted("[VariantsBulkLoad] !!! hbr.Process() finished => hbr._unprocessedRequests.Count:{0}.", hbr._unprocessedRequests.Count.ToString());

foreach (var id2Link in variantsIDs2Links)
{
var requestEntry = _hbr._responses.FirstOrDefault(t => t.Key == id2Link.Key);
if (requestEntry.Failed)
{
throw new Exception(string.Format("[VariantsBulkLoad] request entry with key:'{0}’ failed.", requestEntry.Key));
}
_cacheCustom.SaveHtml(id2Link.Value, requestEntry.Content);
}
_mti.AddLogInfo("[VariantsBulkLoad] Finished.");
}

internal sealed class QtyItem
{
public int Value { get; set; }
public string StoreId { get; set; }
public override string ToString()
{
return $"Value:{Value}, StoreId:{StoreId}";
}
}

internal sealed class CacheCustom
{
private CacheStorage _cacheStorage;
private Dictionary<string, string> _url2Html;
private Dictionary<string, List<QtyItem>> _art2Qty;
private HashSet<string> _productsArtProcessed = new HashSet<string>(StringComparer.InvariantCultureIgnoreCase);
public CacheCustom(CacheStorage cacheStorage)
{
_cacheStorage = cacheStorage;
_url2Html = new Dictionary<string, string>();
_art2Qty = new Dictionary<string, List<QtyItem>>();
}
internal bool IsCached(string url)
{
string html;
if (!_url2Html.TryGetValue(url, out html))
{
if (_cacheStorage.Load(url, out html))
{
_url2Html[url] = html;
}
}
return !string.IsNullOrEmpty(html);
}
internal bool Load(string url, out string html)
{
html = "";
if (IsCached(url))
{
html = _url2Html[url];
return true;
}
return false;
}
internal void SaveHtml(string url, string html)
{
if (!string.IsNullOrEmpty(url))
{
_url2Html[url] = html;
_cacheStorage.Save(url, html);
}
}
internal void SaveQty(string url, QtyItem qty)
{
if (!string.IsNullOrEmpty(url))
{
if (!_art2Qty.TryGetValue(url, out List<QtyItem> qts))
{
qts = new List<QtyItem>();
_art2Qty[url] = qts;
}
qts.Add(qty);
}
}
internal void HtmlCacheClear()
{
_url2Html.Clear();
}
internal bool GetQty(string art, out List<QtyItem> qtyLIst)
{
return _art2Qty.TryGetValue(art, out qtyLIst);
}
internal bool IsProcessed(string art)
{
if (!string.IsNullOrEmpty(art))
{
return _productsArtProcessed.Contains(art);
}
return false;
}

internal void SaveAsProcessed(string art)
{
if (!string.IsNullOrEmpty(art))
{
_productsArtProcessed.Add(art);
}
}
}

//private Dictionary<string, int> _variantId2Qty = new Dictionary<string, int>();
CatalogLoader.Utils.HtmlBulkRequests _hbr = null;
private Dictionary<string, string> VariantsIDs2LinksGet(string variationSection, string productStyles)
{
/*
* akl_12102020
*
* http://crm.catalogloader.com/issues/35427#note-88
* https://www.ikea.com/ru/ru/p/longfell-rabochiy-stul-s-podlokotnikami-gunnared-bezhevyy-chernyy-s29210026/
* "В выгрузку попадают только товары с разным основанием: белый, чёрный.
Разные цвета чехлов не попадают – только бежевый.
Таким образом из 8 вариантов в xml попали 2."
* =========================================================================================================
*
* Нужно перебирать все варианты ("VARIATION_SECTION"), переходить в карточку каждого
* и там уже собирать стили ("PRODUCT_STYLES").
* Иначе, теряются комбинации стилей.
* ==========================================================================================================
*
* akl_18112020
*
* http://crm.catalogloader.com/issues/35427#note-111
* "Предложений должно быть 32, а выгружаеся только 8"
* https://www.ikea.com/ru/ru/p/malm-karkas-krovati-dubovyy-shpon-belenyy-luroy-s39210945/
*
* !!! UP: После перехода в карточку варианта, нужно собирать не только стили, но и сами варианты.
* Иначе, список комбинаций будет неполным.
*
* akl_06122021
* Похоже, переход по ссылке каждого варианта больше не актуален – берем только то,
* что пришло на странице основного товара и на странице каждого стиля.
*/

_mti.AddLogInfo("[VariantsIDs2LinksGet] Started.");
var ids2Variants = new Dictionary<string, Variant>();

// Берем все варианты стилей, если есть…
var stylesId2Variant = new Dictionary<string, Variant>();
if (!string.IsNullOrEmpty(productStyles))
{
foreach (var id2Variant in Ids2VariantsParse(productStyles, typeof(ModelsIkea.Variations.Variation)))
{
if (!_mti.CanContinue())
{
break;
}
stylesId2Variant[id2Variant.Key] = id2Variant.Value;
}
}
// …собираем варианты на странице каждого неактивного стиля…
foreach (var styleId2Variant in stylesId2Variant)
{
if (!_mti.CanContinue())
{
break;
}

if (!styleId2Variant.Value.IsSelected)
{
var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty();
hpl.Load(styleId2Variant.Value.Url);
var variantsNode = hpl.HtmlDoc.DocumentNode.SelectSingleNode(".//div[contains(@class,’js-product-variation-section’)]");
if (variantsNode != null)
{
string VARIATION_SECTION = HttpUtility.HtmlDecode(variantsNode.GetAttributeValue("data-initial-props", ""));
if (!string.IsNullOrEmpty(VARIATION_SECTION))
{
foreach (var id2Variant in Ids2VariantsParse(VARIATION_SECTION, typeof(ModelsIkea.Variations.VariationsSection)))
{
if (!_mti.CanContinue())
{
break;
}
ids2Variants[id2Variant.Key] = id2Variant.Value;
}
}
else
{
//…если вариантов нет, то берем сам стиль…
ids2Variants[styleId2Variant.Key] = styleId2Variant.Value;
}
}
}
}
// …собираем варианты на странице активного стиля…
if (!string.IsNullOrEmpty(variationSection))
{
foreach (var id2Variant in Ids2VariantsParse(variationSection, typeof(ModelsIkea.Variations.VariationsSection)))
{
if (!_mti.CanContinue())
{
break;
}
ids2Variants[id2Variant.Key] = id2Variant.Value;
}
}
else
{
var styleSelectedId2Var = stylesId2Variant.FirstOrDefault(id2v => id2v.Value.IsSelected);
if (!string.IsNullOrEmpty(styleSelectedId2Var.Key))
{
ids2Variants[styleSelectedId2Var.Key] = styleSelectedId2Var.Value;
}
}

_cacheCustom.SaveAsProcessed(id2v.Key);
}
else
{
nonProcessedVariants.Add(id2v.Value);
}
}

List<Variant> varsValid = VariantsAvailabilityValidation(nonProcessedVariants);
_mti.AddLogInfo("[VariantsIDs2LinksGet] Finished.");
return varsValid.Count > 0 ? varsValid.ToDictionary(v => v.Id, v => v.Url) : new Dictionary<string, string>();
}

CatalogLoader.Utils.HtmlBulkRequests HtmlBulkRequestsGet()
{
if (_hbr == null)
{
_hbr = new CatalogLoader.Utils.HtmlBulkRequests();
_hbr.ThreadsNumber = 20;
_hbr.ProxySourceList = "http://ru.proxy-doctor.com/WebApi/CampaignDataLatestGet/1-201705051618-NNCIAPGJEYMVQKGXQRXKFKCOKJLDNMRVBFBQQMLXHPQHYBMDXK?collection-name=ikearu";
_hbr.BanDetectionString = "The following error was encountered while trying to retrieve the URL[next]Method Not Allowed[next]Access Denied[next]非";
//_hbr.RequestAttempts = _gps.GrabberSettings.Settings.RequestAttempts;
_hbr.RequestAttempts = 10;
_hbr.RequestTimeout = _gps.GrabberSettings.Settings.RequestTimeout;
_hbr.CustomRequestUse = true;
}
return _hbr;
}

private List<Variant> VariantsAvailabilityValidation(List<Variant> variants)
{
_mti.AddLogFormatted("[VariantsAvailabilityValidation] Started. variants total:{0}.", variants.Count.ToString());
var varsToProcess = new List<Variant>();
var id2IdNorm = new Dictionary<string, string>();
foreach (var v in variants)
{
if (!_cacheCustom.GetQty(v.Id, out List<QtyItem> qts))
{
varsToProcess.Add(v);
}
}
if (varsToProcess.Count > 0)
{
var bulkRequests = new List<CatalogLoader.Utils.HtmlBulkRequests.HttpWebRequestEntry>();
_mti.AddLogFormatted("[VariantsAvailabilityValidation] !!!varsToProcess.Count:{0}.", varsToProcess.Count.ToString());
var idNorm2Qhtml = new Dictionary<string, string>(varsToProcess.Count);
foreach (var vp in varsToProcess)
{
string idNorm = vp.Id.ToUpper();
if (idNorm.StartsWith("S"))
{
idNorm = idNorm.TrimStart(‘S’);
}
id2IdNorm[vp.Id] = idNorm;

//var aviTest = ProductAvailabilityInfoGet(_gps.GrabberSettings.UserParameterGet("StoreId"), id2IdNorm[vp.Id]);

//string qUrl = string.Format("https://api.ingka.ikea.com/cia/availabilities/ru/us?itemNos={0}&expand=StoresList,Restocks", idNorm);
string qUrl = string.Format(_ikeaSettings.RegionSettings.ProductAvailabilityUrlTemplate, idNorm);
string qHtml;
if (!_cacheCustom.Load(qUrl, out qHtml))
{
var r = HttpWebRequest.Create(qUrl) as HttpWebRequest;
r.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.45 Safari/537.36";
r.Accept = "application/json;version=2";
r.Headers["X-Client-Id"] = "b6c117e5-ae61-4ef5-b4cc-e0b1e37f0631";
r.Headers["accept-encoding"] = "gzip, deflate, br";
r.CookieContainer = _gps.m_cookies_con;
bulkRequests.Add(new CatalogLoader.Utils.HtmlBulkRequests.HttpWebRequestEntry()
{
Key = vp.Id,
Request = r
});
}
else
{
idNorm2Qhtml[idNorm] = qHtml;
}
}

var hbr = HtmlBulkRequestsGet();
_mti.AddLogFormatted("[VariantsAvailabilityValidation] !!! hbr.Process() started => hbr._requests.Count:{0}.", bulkRequests.Count.ToString());
hbr.Process(bulkRequests);
_mti.AddLogFormatted("[VariantsAvailabilityValidation] !!! hbr.Process() finished => hbr._unprocessedRequests.Count:{0}.", hbr._unprocessedRequests.Count.ToString());

foreach (var vp in varsToProcess)
{
string qHtml = "";
var requestEntry = _hbr._responses.FirstOrDefault(t => t.Key == vp.Id);
if (requestEntry != null)
{
if (requestEntry.Failed)
{
throw new Exception(string.Format("[VariantsAvailabilityValidation] request entry with key:'{0}’ failed.", requestEntry.Key));
}
else
{
qHtml = requestEntry.Content;
_cacheCustom.SaveHtml(requestEntry.Request.Address.AbsoluteUri, requestEntry.Content);
}

}
else
{
idNorm2Qhtml.TryGetValue(id2IdNorm[vp.Id], out qHtml);
}

foreach (var storeId in _ikeaSettings.StoreIDs)
{
var avi = ProductAvailabilityInfoGet(storeId, id2IdNorm[vp.Id], qHtml);
_cacheCustom.SaveQty(vp.Id, new QtyItem() { Value = avi.Qty, StoreId = storeId });
}
}
}

List<Variant> validVars = null;
if (_ikeaSettings.ProductQtyMin > 0)
{
validVars = new List<Variant>();
foreach (var v in variants)
{
if (_cacheCustom.GetQty(v.Id, out List<QtyItem> qtyList))
{
int qtySum = qtyList.Sum(q => q.Value);
if (qtySum < _ikeaSettings.ProductQtyMin)
{
_mti.AddLogWarning(string.Format("[VariantsAvailabilityValidation] Variant ‘{0}’ processing interrupt -> current quantity ({1}) < minimum allowable value ({2})", v.Id, qtySum.ToString(), _ikeaSettings.ProductQtyMin.ToString()));
_cacheCustom.SaveAsProcessed(v.Id);
continue;
}
}
validVars.Add(v);
}
}
else
{
validVars = variants;
}
_mti.AddLogFormatted("[VariantsAvailabilityValidation] Finished => valid variants/variants:{0}/{1}.", validVars.Count.ToString(), variants.Count.ToString());
return validVars;
}

private Dictionary<string, Variant> Ids2VariantsParse(string html, Type type)
{
_mti.AddLogInfo("<<<<< Ids2VariantsParse started >>>>>");
if (string.IsNullOrEmpty(html))
{
return new Dictionary<string, Variant>();
}
var ids2Vars = new Dictionary<string, Variant>();
try
{
if (type == typeof(ModelsIkea.Variations.VariationsSection))
{
var j = JsonConvert.DeserializeObject<ModelsIkea.Variations.VariationsSection>(html);
foreach (var vrnt in j.variations)
{
var vrntOptions = vrnt.allOptions.Count > vrnt.options.Count ? vrnt.allOptions : vrnt.options;
foreach (var opt in vrntOptions)
{
if (!string.IsNullOrEmpty(opt.linkId) && !string.IsNullOrEmpty(opt.url))
{

ids2Vars[opt.linkId] = new Variant() { Id = opt.linkId, Url = opt.url, IsSelected = opt.isSelected, Price = opt.priceProps.revampPrice.PriceDoubleGet() };
}
}
}
}
else if (type == typeof(ModelsIkea.Variations.Variation))
{
var json = JObject.Parse(html);
var j = JsonConvert.DeserializeObject<ModelsIkea.Variations.Variation>(json["variationStyles"].First.ToString());
var options = j.allOptions.Count > j.options.Count ? j.allOptions : j.options;
foreach (var opt in options)
{
if (!string.IsNullOrEmpty(opt.linkId) && !string.IsNullOrEmpty(opt.url))
{
ids2Vars[opt.linkId] = new Variant() { Id = opt.linkId, Url = opt.url, IsSelected = opt.isSelected, Price = opt.priceProps.revampPrice.PriceDoubleGet() };
}
}
}

}
catch (Exception ex)
{
_mti.AddLogError(ex);
}

_mti.AddLogInfo("<<< variants count: " + ids2Vars.Count.ToString());
return ids2Vars;
}

private Dictionary<string, string> VariantsIDs2LinksParse(string html, Type type)
{
_mti.AddLogInfo("<<<<< VariantsIDs2LinksParse started >>>>>");
if (string.IsNullOrEmpty(html))
{
return new Dictionary<string, string>();
}

}
catch (Exception ex)
{
_mti.AddLogError(ex);
}

_mti.AddLogInfo("<<< variants IDs2Links count: " + ids2Links.Count.ToString());
return ids2Links;
}

private HttpMethodParameters _hmpForVariantsLoad = new HttpMethodParameters()
{
MethodName = "GET",
Accept = "application/vnd.ikea.iows+json;version=2.0",
httpHeaders = new Dictionary<string, string>()
{
/*
* Headers logic info script.
* https://ikeasources.ru/b/m2.prod.js?v=156940268994123778112121
*/
//{ "Sec-Fetch-Mode", "cors" },
//{ "Consumer", "DEVRU" },
{ "Consumer", "MAMMUT" },
//{ "Contract", "39799" },
{ "Contract", "37249" },
{ "accept-encoding", "gzip, deflate, br" },
}
};
private bool VariantProcessing(Product productForProcessing)
{
_mti.AddLogInfo("<<<<< VariantProcessing started >>>>>");
_mti.AddLogInfo("<<< product for processing type:" + productForProcessing.GetType());
_mti.AddLogInfo("<<< product for processing id:" + productForProcessing.Art);

string artPrepared = productForProcessing.Art.ToUpper();
if (artPrepared.StartsWith("S"))
{
artPrepared = artPrepared.TrimStart(‘S’);
}

if (!_cacheCustom.GetQty(productForProcessing.Art, out List<QtyItem> qtyList))
{
qtyList = new List<QtyItem>();
string qtyHtml;
_cacheCustom.Load(string.Format("https://api.ingka.ikea.com/cia/availabilities/ru/us?itemNos={0}&expand=StoresList,Restocks", artPrepared), out qtyHtml);
foreach (var storeId in _ikeaSettings.StoreIDs)
{
var avi = ProductAvailabilityInfoGet(storeId, artPrepared, qtyHtml);
if (avi.Qty < 1)
{
avi.Qty = 0;
}
var qtyItem = new QtyItem() { Value = avi.Qty, StoreId = storeId };
_cacheCustom.SaveQty(productForProcessing.Art, qtyItem);
qtyList.Add(qtyItem);
}
}

int qty = qtyList.Sum(q => q.Value);
productForProcessing.Quantity = qty.ToString();

foreach (var item in qtyList)
{
productForProcessing.AddDynamicAttribute($"QTY[{item.StoreId}]", item.Value.ToString());
}

_mti.AddLogInfo($"[VariantProcessing] Variant ‘{productForProcessing.Art}’ QTY list:{string.Join("[next]", qtyList.Select(q => q.ToString()))}");

//if (qty == 0)
if (_ikeaSettings.ProductQtyMin > 0 && qty < _ikeaSettings.ProductQtyMin)
{
_mti.AddLogWarning(string.Format("[VariantProcessing] Variant ‘{0}’ processing interrupt -> current quantity ({1}) < minimum allowable value ({2})",
productForProcessing.Art, qty, _ikeaSettings.ProductQtyMin));

return true;
}

/*
string productHtml = "";
if (!_url2ProductPageHtml.TryGetValue(productForProcessing.Url, out productHtml))
{
var loader = _gps.Proxy.GetHtmlPageLoaderEmpty();
loader.Load(productForProcessing.Url);
_url2ProductPageHtml[productForProcessing.Url] = productHtml = loader.Content;
}
*/

//string variantJsonUrl = string.Format("https://iows.ikea.com/retail/iows/us/en/catalog/items/{0},{1}?ignoreErrors=true", pathSegment.ToUpper(), artPrepared);
//var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty();
//if (!hpl.Load(variantJsonUrl, true, _hmpForVariantsLoad))
//{
// _mti.AddLogWarning("!!! Can’t download variant JSON.");
// return false;
//}

//var json = JObject.Parse(hpl.Content);
//var root = json["RetailItemComm"];

//// https://www.ikea.com/us/en/p/hauga-upholstered-bed-frame-lofallet-beige-30490484/
//if (root == null)
//{

//ProductPopulateByProductHtml(_url2ProductPageHtml[productForProcessing.Url], productForProcessing);
string html;
if (!_cacheCustom.Load(productForProcessing.Url, out html))
{
_mti.AddLogWarning(string.Format("[VariantProcessing] Variant ‘{0}’ processing interrupt -> can’t find product page html.", productForProcessing.Art));
}
ProductPopulateByProductHtml(html, productForProcessing);

//}
//else
//{
// var priceToken = root.SelectToken("$.RetailItemCommPriceList.RetailItemCommPrice");
// if (priceToken != null && priceToken.HasValues)
// {
// if (priceToken is JArray)
// {
// bool isNewLowerPrice = false;
// foreach (var item in priceToken)
// {
// var price = item["Price"].Value<string>("$");
// var retailPriceType = item["RetailPriceType"].Value<string>("$");
// if (retailPriceType.Equals("RegularSalesUnitPrice", StringComparison.InvariantCultureIgnoreCase))
// {
// if (!isNewLowerPrice)
// {
// var reasonCode = item["ReasonCode"];
// if (reasonCode != null && reasonCode.Value<string>("$").Equals("NewLowerPrice", StringComparison.InvariantCultureIgnoreCase))
// {
// isNewLowerPrice = true;
// }
// productForProcessing.Price = price;
// }
// }
// if (retailPriceType.Equals("IKEAFamilySalesUnitPrice", StringComparison.InvariantCultureIgnoreCase))
// {
// productForProcessing.PriceOld = productForProcessing.Price;
// productForProcessing.Price = price;
// }
// }
// }
// else
// {
// productForProcessing.Price = priceToken["Price"].Value<string>("$");
// }
// }

// var optionsToken = root.SelectToken("$.GPRCommSelectionCriteriaSelectionList.GPRCommSelectionCriteriaSelection");
// if (optionsToken != null && optionsToken.HasValues)
// {
// if (optionsToken is JArray)
// {
// foreach (var option in optionsToken)
// {
// var name = option[/*"SelectionCriteriaCode"*/"SelectionCriteriaName"].Value<string>("$");
// var val = option["SelectionCriteriaValue"].Value<string>("$");
// if (!string.IsNullOrEmpty(name) && !string.IsNullOrEmpty(val))
// {
// if (val == "-")
// {
// continue;
// }
// var nameChars = name.Trim().ToCharArray();
// nameChars[0] = char.ToUpper(nameChars[0]);
// productForProcessing.AddDynamicAttribute(new string(nameChars), val.Trim());
// }
// }
// }
// else
// {
// var name = optionsToken[/*"SelectionCriteriaCode"*/"SelectionCriteriaName"].Value<string>("$");
// var val = optionsToken["SelectionCriteriaValue"].Value<string>("$");
// if (!string.IsNullOrEmpty(name) && !string.IsNullOrEmpty(val))
// {
// if (val != "-")
// {
// var nameChars = name.Trim().ToCharArray();
// nameChars[0] = char.ToUpper(nameChars[0]);
// productForProcessing.AddDynamicAttribute(new string(nameChars), val.Trim());
// }
// }
// }
// }

// Dictionary<string, string> productMeasuresNames2Values = ProductMeasuresParse(root);
// foreach (var n2v in productMeasuresNames2Values)
// {
// productForProcessing.AddDynamicAttribute(n2v.Key, n2v.Value);
// }

// //List<PackageLegacy> packages = PackageMeasuresParse(productHtml);
// //PackagesAttributesProcessing(productForProcessing, packages);

// var productNameTokenTitle2Val = new Dictionary<string, string>();
// var productName = root["ProductName"];
// if (productName != null)
// {
// productNameTokenTitle2Val["ProductName"] = productName.Value<string>("$").Trim();
// //productForProcessing.Name = productName.Value<string>("$").Trim();
// }
// var productTypeName = root["ProductTypeName"];
// if (productTypeName != null)
// {
// productNameTokenTitle2Val["ProductTypeName"] = productTypeName.Value<string>("$").Trim();
// //productForProcessing.Name += " " + productTypeName.Value<string>("$").Trim();
// }
// var validDesignText = root["ValidDesignText"];
// if (validDesignText != null)
// {
// productNameTokenTitle2Val["ValidDesignText"] = validDesignText.Value<string>("$").Trim();
// }
// var itemMeasureReferenceTextMetric = root["ItemMeasureReferenceTextMetric"];
// if (itemMeasureReferenceTextMetric != null)
// {
// productNameTokenTitle2Val["ItemMeasureReferenceTextMetric"] = itemMeasureReferenceTextMetric.Value<string>("$").Trim();
// }

// productForProcessing.Name = productNameTokenTitle2Val["ProductName"] + " "
// + string.Join(", ", productNameTokenTitle2Val.Where(kvp => kvp.Key != "ProductName").Select(kvp => kvp.Value));

// productForProcessing.FullDescription = DescriptionFullBuild(root);
// //productForProcessing.FullDescription += DescriptionOfPackagesBuild(packages);

// /*
// var attachmentUrlTokens = root.SelectTokens("$.RetailItemCommAttachmentList..AttachmentUrl");
// if (attachmentUrlTokens != null)
// {
// var links = from t in attachmentUrlTokens
// let url = t.Value<string>("$")
// where !string.IsNullOrEmpty(url)
// select TextUtils.UrlNewCompose(_domain, url);

// if (links.Any())
// {
// productForProcessing.AddDynamicAttribute("Инструкция", string.Join(",", links));
// }
// }
// */
//}

if (productForProcessing.ImagesCount == 0)
{
ImagesFill(productForProcessing);
}

//var avi = ProductAvailabilityInfoGet(_gps.GrabberSettings.UserParameterGet("StoreId"), artPrepared);
//productForProcessing.Quantity = avi.Qty.ToString();
//productForProcessing.AddDynamicAttribute("Коробки", BoxAttributeValueBuild(avi, packages));
return true;
}

private void ProductPopulateByProductHtml(string productHtml, Product product)
{
_mti.AddLogFormatted("[ProductPopulateByProductHtml] Started. Art:{0}.", product.Art);

if (string.IsNullOrEmpty(productHtml))
{
_mti.AddLogWarning("[ProductPopulateByProductHtml] Product html is empty.");
return;
}

product.Name = ProductNameBuild(productHtml);

var hDoc = new HtmlDocument();
hDoc.LoadHtml(productHtml);

var nodes = hDoc.DocumentNode.SelectNodes(".//div[@data-initial-props]");
if (nodes != null)
{
var blockClass2InitProps = nodes.ToDictionary(n => n.Attributes["class"].Value, n => n.Attributes["data-initial-props"].Value);
foreach (var block in blockClass2InitProps)
{
var html = HttpUtility.HtmlDecode(block.Value);
var j = JObject.Parse(html);

if (block.Key.IndexOf("pip-price-package", StringComparison.InvariantCultureIgnoreCase) >= 0)
{
var price = j["price"]["mainPriceProps"]["price"];
product.Price = string.Format("{0}{1}{2}", price.Value<string>("integer"), j["price"]["mainPriceProps"].Value<string>("separator"), price.Value<string>("decimals"));
}
else if (block.Key.IndexOf("revamp-product-styles", StringComparison.InvariantCultureIgnoreCase) >= 0)
{
var variant = JsonConvert.DeserializeObject<ModelsIkea.Variations.Variation>(html);
var options = variant.allOptions.Count > variant.options.Count ? variant.allOptions : variant.options;
foreach (var opt in options)
{
if (opt.isSelected && !string.IsNullOrEmpty(opt.title))
{
product.AddDynamicAttribute(variant.title, opt.title);
}
}
}
else if (block.Key.IndexOf("product-variation-section", StringComparison.InvariantCultureIgnoreCase) >= 0)
{
var variations = j.SelectToken("variations");
if (variations != null)
{
variations = variations as JArray;
if (variations != null && variations.HasValues)
{
foreach (JObject v in variations)
{
var variant = v.ToObject<ModelsIkea.Variations.Variation>();
var options = variant.allOptions.Count > variant.options.Count ? variant.allOptions : variant.options;
foreach (var opt in options)
{
if (opt.isSelected && !string.IsNullOrEmpty(opt.title))
{
product.AddDynamicAttribute(variant.title, opt.title);
}
}
}
}
}
}
else if (block.Key.IndexOf("product-information-section", StringComparison.InvariantCultureIgnoreCase) >= 0)
{
var dimName2Val = new Dictionary<string, string>();
try
{
var dimensionProps = j.Value<JObject>("dimensionProps");
if (dimensionProps != null)
{
var dims = dimensionProps.Value<JArray>("dimensions");
if (dims != null && dims.HasValues)
{
foreach (JObject d in dims)
{
var n = d.Value<string>("name");
var v = d.Value<string>("measure");
if (!string.IsNullOrEmpty(n) && !string.IsNullOrEmpty(v))
{
dimName2Val[n.Trim()] = v.Trim();
}
}
}
}
}
catch { }

foreach (var n2v in dimName2Val)
{
product.AddDynamicAttribute(n2v.Key, n2v.Value);
}

var pkgs = PackageMeasuresParse(html, _ikeaSettings.Region == Region.EN ? MeasureSystem.US_STANDARD : MeasureSystem.METRIC);

if (_ikeaSettings.Region == Region.EN)
{
var w = PackageHelper.WeightTotalUsStandardValCalc(pkgs);
if (_ikeaSettings.PackagesWeightTotalAsAttribute)
{
product.AddDynamicAttribute("Package weight", string.Format("{0}{1}", w.Key > 0 ? w.Key.ToString() + " lb " : "",
w.Value > 0 ? w.Value.ToString() + " oz" : ""));
}
}
else
{
if (_ikeaSettings.ProductSizesAsPackagesSizesTotal)
{
product.Weight = PackageHelper.MeasureTotalValCalc(pkgs, MeasureType.WEIGHT, _ikeaSettings.Region).ToString();
product.Height = PackageHelper.MeasureTotalValCalc(pkgs, MeasureType.HEIGHT, _ikeaSettings.Region).ToString();
product.Width = PackageHelper.MeasureTotalValCalc(pkgs, MeasureType.WIDTH, _ikeaSettings.Region).ToString();
product.Depth = PackageHelper.MeasureTotalValCalc(pkgs, MeasureType.LENGTH, _ikeaSettings.Region).ToString();
}
}

product.FullDescription = DescriptionFullBuild(j.Value<JObject>("productDetailsProps"), dimensions: dimName2Val);
}
}
}
}

private string ProductNameBuild(string productHtml)
{
if (!string.IsNullOrEmpty(productHtml))
{
var hdoc = new HtmlDocument();
hdoc.LoadHtml(productHtml);
//var node_1 = hdoc.DocumentNode.SelectSingleNode(".//div[@id=’name’]");
//string n1 = NodeValGet(node_1, ".//strong", @"[\r|\t|\n|\s]{2,}[—>] ");
//var node_2 = hdoc.DocumentNode.SelectSingleNode(".//h1");
//string n2 = NodeValGet(node_2, ".//strong", @"</div><div[^>]*?>[—>] [next]<[^>]*?>[next]\s{2,}[—>] ");
//return string.Format("{0} {1}", n1.Trim(), HttpUtility.HtmlDecode(n2.Trim())).Trim();
var node_2 = hdoc.DocumentNode.SelectSingleNode(".//h1");
var parts = new List<string>();
if (node_2 != null)
{
foreach (var n in node_2.ChildNodes)
{
if (!string.IsNullOrEmpty(n.InnerText))
{
parts.Add(n.InnerText.Trim());
}
}
}
return HttpUtility.HtmlDecode(string.Join(" ", parts));
}
return string.Empty;
}
private string NodeValGet(HtmlNode node, string removeAllInnerNodesXpath = null, string replaceInnerHtmlRegex = null)
{
if (node != null)
{
if (!string.IsNullOrEmpty(removeAllInnerNodesXpath))
{
var rems = node.SelectNodes(removeAllInnerNodesXpath);
if (rems != null)
{
foreach (var r in rems)
{
r.Remove();
}
}
var val = node.InnerHtml;
if (!string.IsNullOrEmpty(replaceInnerHtmlRegex))
{
var patterns = UtilSmall.GetParts(replaceInnerHtmlRegex);
foreach (var p in patterns)
{
var fromTo = UtilSmall.GetParts(p, "[—>]");
if (fromTo.Length == 1)
{
fromTo = new string[2] { fromTo[0], "" };
}
val = Regex.Replace(val, fromTo[0], fromTo[1]);
}
}
return val;
}
}
return string.Empty;
}

private string BoxAttributeValueBuild(AvailabilityInfo avi, List<PackageLegacy> packages, string propDelim = ";", string itemDelim = "[next]")
{
var vals = from s in avi.SalesLocations
join p in packages on s.ItemNo equals p.ItemNo
select string.Format("product_set_name:{1}{0}product_count:{2}{0}product_set_ean:{3}{0}row:{4}{0}place:{5}",
propDelim,
p.ProductName,
p.PackagesQty,
p.ItemNo,
s.Aisle,
s.Bin);

return string.Join(itemDelim, vals);
}

private string DescriptionOfPackagesBuild(List<PackageLegacy> packages)
{
var sBuilder = new StringBuilder();
sBuilder.Append("<div><strong>Packaging</strong></div>");
if (packages.Count > 1)
{
sBuilder.AppendFormat("<h5>This product comes as {0} packages.</h5>", packages.Sum(p => p.PackagesQty).ToString());
}

string idCurrent = "default";
foreach (var pkg in packages)
{
sBuilder.Append("<div>");
string id = IDCreate(pkg.ItemNo);
if (idCurrent != id)
{
idCurrent = id;
sBuilder.AppendFormat("<div><h5>{0}</h5><p>{1}</p><p><span>Article Number</span><br/><span>{2}</span></p></div>",
pkg.ProductName, pkg.ProductTypeName, id);
}
else
{ sBuilder.Append("<br/>"); }

foreach (var measureN2V in pkg.MeasuresName2Val)
{
sBuilder.AppendFormat("<div>{0}: {1}</div>", PackageHelper.PackageSizeNameTranslate(measureN2V.Key), measureN2V.Value);
}
sBuilder.AppendFormat("<div>{0}: {1}</div>", "Package(s)", pkg.PackagesQty.ToString());

sBuilder.Append("</div>");
}
return sBuilder.ToString();
}
private void PackagesAttributesProcessing(Product product, List<PackageLegacy> packages)
{
/*
* http://crm.catalogloader.com/issues/35219
4. Вес товара и габариты упаковки
Пояснение:
— Кол-во товаров в упаковке сделать характеристикой; Ок.
— Расчёт габаритов;
Ширина. Берём максимальное значение
Высота. Суммируем
Длина. Берём максимальное значение
Полученное выводим в характеристики.
Это будет возможно, если на всех карточках товарах одинаково оформлены габариты
Пробуем. Получится — хорошо, не получится — ладно.
— Расчёт объёма
Рассчитать объём и вывести характеристикой Напишите формулу расчета.
Формула расчёта объёма:
Полученные габариты суммируем:
58*21*88 = 107 184 см3.
Получаем объём в м3. 107 184 * 0,000001 = 0,107184 м3
*/
product.AddDynamicAttribute("Кол-во упаковок", packages.Sum(p => p.PackagesQty).ToString());
double weightTotal = PackageHelper.MeasureTotalValCalc(packages, MeasureType.WEIGHT, _ikeaSettings.Region);
product.SetAttributeValue("SYS_PRODUCT_WEIGHT", string.Format("{0:N2}", weightTotal));

double heightTotal = PackageHelper.MeasureTotalValCalc(packages, MeasureType.HEIGHT, _ikeaSettings.Region);
product.AddDynamicAttribute(PackageHelper.PackageSizeNameTranslate(MeasureType.HEIGHT.ToString()), string.Format("{0:N2}", heightTotal));

double widthMax = PackageHelper.MeasureMaxValueGet(packages, MeasureType.WIDTH, _ikeaSettings.Region);
product.AddDynamicAttribute(PackageHelper.PackageSizeNameTranslate(MeasureType.WIDTH.ToString()), string.Format("{0:N2}", widthMax));

double lengthMax = PackageHelper.MeasureMaxValueGet(packages, MeasureType.LENGTH, _ikeaSettings.Region);
product.AddDynamicAttribute(PackageHelper.PackageSizeNameTranslate(MeasureType.LENGTH.ToString()), string.Format("{0:N2}", lengthMax));

double pakageVal = heightTotal * widthMax * lengthMax * 0.000001;
product.AddDynamicAttribute("Объем, м3", string.Format("{0:N2}", pakageVal));
}

private List<PackageLegacy> PackageMeasuresParse(string productHtml, MeasureSystem measureSystem = MeasureSystem.METRIC)
{
_mti.AddLogInfo("[PackageMeasuresParse] Started.");
var packages = new List<PackageLegacy>();
if (!string.IsNullOrEmpty(productHtml))
{
try
{
//string jsonStr = HtmlValueGet(productHtml, ".//div[contains(@class,’js-product-information-section range-revamp-product-information-section’)]", "data-initial-props");
string jsonStr = productHtml;

if (!string.IsNullOrEmpty(jsonStr))
{
jsonStr = HttpUtility.HtmlDecode(jsonStr);
var json = JsonConvert.DeserializeObject<ModelsIkea.ProductProps.Props>(jsonStr);
var packaging = json.DimensionPropsObject != null && json.DimensionPropsObject.packaging != null ? json.DimensionPropsObject.packaging : null;

if (packaging == null)
{
_mti.AddLogInfo("[PackageMeasuresParse] json.DimensionPropsObject.packaging == null ? TRUE");
packaging = json.productDetailsProps != null && json.productDetailsProps.accordionObject != null ? json.productDetailsProps.accordionObject.packaging : null;
if (packaging != null)
{
_mti.AddLogInfo("[PackageMeasuresParse] packaging IS json.productDetailsProps.accordionObject.packaging");
}
}
else
{
_mti.AddLogInfo("[PackageMeasuresParse] packaging IS json.DimensionPropsObject.packaging");
}

if (packaging != null && packaging.contentProps != null && packaging.contentProps.packages != null)
{
foreach (var pck in packaging.contentProps.packages)
{
if (pck.measurements == null || pck.measurements.Length == 0)
{
continue;
}
foreach (var m in pck.measurements)
{
PackageLegacy package = null;
var m2vr = new Dictionary<string, string>();
foreach (var msrm in m)
{
if (string.IsNullOrEmpty(msrm.label) || string.IsNullOrEmpty(msrm.value))
{
continue;
}
//if (package == null)
//{
// package = new PackageLegacy();
//}
m2vr[msrm.label.ToUpperInvariant()] = msrm.value;
//string valNormal = Regex.Replace(msrm.value, @"\D+?$", "");
//package.MeasuresName2Val[MeasureNameLegacyGet(msrm.label)] = UtilSmall.ConvertToDouble(valNormal);

}
if (package == null)
{
package = new PackageLegacy(m2vr, measureSystem);
}
if (package != null)
{
package.PackagesQty = pck.quantity.value;
package.ItemNo = pck.articleNumber.value;
package.ProductName = pck.name;
package.ProductTypeName = pck.typeName;

packages.Add(package);
}
}
}
}
else
{
_mti.AddLogInfo("[PackageMeasuresParse] json.productDetailsProps.accordionObject.packaging == null ? TRUE");
}
}
}
catch (Exception ex)
{
_mti.AddLogError("PackageMeasuresParse() failed.", ex);
}
}
_mti.AddLogInfo("[PackageMeasuresParse] Finished.");
return packages;
}
private string MeasureNameLegacyGet(string nameModern)
{
string ptrn = nameModern.ToUpperInvariant();
switch (ptrn)
{
case "ШИРИНА":
return "WIDTH";
case "ВЫСОТА":
return "HEIGHT";
case "ДЛИНА":
return "LENGTH";
case "ВЕС":
return "WEIGHT";
case "ДИАМЕТР":
return "DIAMETER";
default:
return nameModern.ToUpperInvariant();
}
}
private string HtmlValueGet(string html, string nodeXpath, string attributeName = null, bool plainTxt = false)
{
string val = "";
try
{
var hdoc = new HtmlDocument();
hdoc.LoadHtml(html);
var node = hdoc.DocumentNode.SelectSingleNode(nodeXpath);
if (node != null)
{
if (string.IsNullOrEmpty(attributeName))
{
val = plainTxt ? node.InnerText : node.OuterHtml;
}
else
{
val = node.GetAttributeValue(attributeName, "");
}
}
}
catch (Exception ex)
{
_mti.AddLogError(ex);
}
return val;
}

private void ImagesFill(Product productForProcessing)
{
string imgsJson = productForProcessing.GetAttributeValue("MEDIA_GRID");
if (string.IsNullOrEmpty(imgsJson))
{
string html = "";
/*
if (!_url2ProductPageHtml.TryGetValue(productForProcessing.Url, out html))
{
var hplImgs = _gps.Proxy.GetHtmlPageLoaderEmpty();
if (hplImgs.Load(productForProcessing.Url))
{
_url2ProductPageHtml[productForProcessing.Url] = html = hplImgs.Content;
}
}
*/
if (!_cacheCustom.Load(productForProcessing.Url, out html))
{
var hplImgs = _gps.Proxy.GetHtmlPageLoaderEmpty();
hplImgs.Load(productForProcessing.Url);
}

if (!string.IsNullOrEmpty(html))
{
var hdoc = new HtmlDocument();
hdoc.LoadHtml(html);
//imgsJson = TextUtils.GetHtmlValue(hdoc, ".//div[contains(@class,’js-range-media-grid range-revamp-media-grid’)]", "data-initial-props", true, 0);
imgsJson = TextUtils.GetHtmlValue(hdoc, ".//div[contains(@class,’js-range-media-grid’)]", "data-initial-props", true, 0);
}
}
if (!string.IsNullOrEmpty(imgsJson))
{
var imgs = new List<ImageClp>();
var mediaLst = new List<ModelsIkea.Media.MediaListBase>();
try
{
var j = JsonConvert.DeserializeObject<ModelsIkea.Media.MediaGrid>(imgsJson);
mediaLst = j.fullMediaList.Any() ? j.fullMediaList.ToList<ModelsIkea.Media.MediaListBase>()
: j.mediaList.ToList<ModelsIkea.Media.MediaListBase>();
}
catch (Exception ex)
{
_mti.AddLogError(ex);
}
foreach (var media in mediaLst)
{
if (!media.type.Equals("image", StringComparison.InvariantCultureIgnoreCase)
|| media.content == null
|| string.IsNullOrEmpty(media.content.imageFileName)
|| string.IsNullOrEmpty(media.content.url))
{
continue;
}

imgs.Add(new ImageClp() { FileName = media.content.imageFileName, Url = media.content.url });
}
productForProcessing.ImageSafeAdd(imgs);
}
}
private Dictionary<string, string> ProductMeasuresParse(JToken root)
{
var measuresNames2Vals = new Dictionary<string, string>();
var itemMeasureToken = root.SelectToken("$.RetailItemCommMeasureList.RetailItemCommMeasure");
if (itemMeasureToken != null && itemMeasureToken.HasValues)
{
if (itemMeasureToken is JArray)
{
foreach (var pkg in itemMeasureToken)
{
var nameOrig = pkg["ItemMeasureTypeName"].Value<string>("$");
var val = pkg["ItemMeasureTextMetric"].Value<string>("$");
if (string.IsNullOrEmpty(nameOrig) || string.IsNullOrEmpty(val))
{
continue;
}
measuresNames2Vals[nameOrig.Trim()] = val.Trim();
}
}
else
{
var nameOrig = itemMeasureToken["ItemMeasureTypeName"].Value<string>("$");
var val = itemMeasureToken["ItemMeasureTextMetric"].Value<string>("$");
if (!string.IsNullOrEmpty(nameOrig) || !string.IsNullOrEmpty(val))
{
measuresNames2Vals[nameOrig.Trim()] = val.Trim();
}
}
}
return measuresNames2Vals;
}

internal class AvailabilityInfo
{
internal string ItemNo { get; set; }
internal int Qty { get; set; }
internal List<SalesLocationInfo> SalesLocations { get; set; }
internal AvailabilityInfo()
{
Qty = -1;
SalesLocations = new List<SalesLocationInfo>();
}
}

internal class SalesLocationInfo
{
internal string ItemNo { get; set; }
internal string Aisle { get; set; }
internal string Bin { get; set; }
}

internal class PackageLegacy
{
internal Dictionary<string, double> MeasuresName2Val { get; private set; }
internal Dictionary<string, string> MeasuresName2ValRaw { get; private set; }
internal string ProductName { get; set; }
internal string ItemNo { get; set; }
internal int PackagesQty { get; set; }
public string ProductTypeName { get; internal set; }
public MeasureSystem MeasureSystem { get; private set; }

internal PackageLegacy(Dictionary<string, string> measuresName2ValRaw, MeasureSystem measureSystem = MeasureSystem.METRIC)
{
PackagesQty = 1;
MeasuresName2Val = new Dictionary<string, double>();
MeasuresName2ValRaw = measuresName2ValRaw;
MeasureSystem = measureSystem;
Init();
}

private void Init()
{
if (MeasuresName2ValRaw == null)
{
MeasuresName2ValRaw = new Dictionary<string, string>();
}
foreach (var n2rv in MeasuresName2ValRaw)
{
if (MeasureSystem == MeasureSystem.METRIC)
{
string valNormal = Regex.Replace(n2rv.Value, @"\D+?$", "");
MeasuresName2Val[n2rv.Key] = UtilSmall.ConvertToDouble(valNormal);
}
}
}

internal string ToHtml()
{
var sBuilder = new StringBuilder("<package>");
foreach (var n2v in MeasuresName2Val)
{
string nameToLower = n2v.Key.ToLowerInvariant();
double valueNormal = n2v.Value;
if (nameToLower != "weight")
{
valueNormal = valueNormal / 100;
}
sBuilder.AppendFormat("<{0}>{1}</{0}>", nameToLower, valueNormal.ToString());
}
sBuilder.AppendFormat("<{0}>{1}</{0}>", "sku", ItemNo);
sBuilder.AppendFormat("<{0}>{1}</{0}>", "quantity", PackagesQty);
sBuilder.Append("</package>");
return sBuilder.ToString();
}
}

internal enum MeasureType
{
WIDTH, HEIGHT, LENGTH, WEIGHT, DIAMETER
}
internal enum MeasureSystem
{ METRIC, US_STANDARD }

internal static class PackageHelper
{
internal static double MeasureMaxValueGet(List<PackageLegacy> packages, MeasureType type, Region r)
{
double valMax = 0;
foreach (var p in packages)
{
double val;
//if (p.MeasuresName2Val.TryGetValue(type.ToString(), out val))
if (p.MeasuresName2Val.TryGetValue(PackageSizeNameTranslate(type, r), out val))
{
if (valMax < val)
{
valMax = val;
}
}
}
return valMax;
}
internal static double MeasureTotalValCalc(List<PackageLegacy> packages, MeasureType type, Region r)
{
double valTotal = 0;
foreach (var p in packages)
{
double val;
//if (p.MeasuresName2Val.TryGetValue(type.ToString(), out val))
if (p.MeasuresName2Val.TryGetValue(PackageSizeNameTranslate(type, r), out val))
{
valTotal += val * p.PackagesQty;
}
}
return valTotal;
}
internal static KeyValuePair<int, int> WeightTotalUsStandardValCalc(List<PackageLegacy> packages)
{
int lbs = 0;
int ozs = 0;
string ptrn = @"((?<lb>\d+)\slb)?\s*((?<oz>\d+)\s*oz)?";
foreach (var p in packages)
{
if (p.MeasureSystem == MeasureSystem.US_STANDARD)
{
string val;
if (p.MeasuresName2ValRaw.TryGetValue(PackageSizeNameTranslate(MeasureType.WEIGHT, Region.EN), out val))
{
if (Regex.IsMatch(val, ptrn, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase))
{
var grps = Regex.Match(val, ptrn, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase).Groups;
if (grps["lb"] != null && !string.IsNullOrEmpty(grps["lb"].Value))
{
lbs += int.Parse(grps["lb"].Value);
}
if (grps["oz"] != null && !string.IsNullOrEmpty(grps["oz"].Value))
{
ozs += int.Parse(grps["oz"].Value);
lbs += ozs / 16; // 1 lb = 16 oz
ozs %= 16;
}
}
}
}
}

return new KeyValuePair<int, int>(lbs, ozs);
}
internal static string PackageSizeNameTranslate(string source)
{
if (string.IsNullOrEmpty(source))
{
return string.Empty;
}
switch (source)
{
case "WEIGHT":
return "Вес упаковки, кг";
case "HEIGHT":
return "Высота упаковки, см";
case "LENGTH":
return "Длина упаковки, см";
case "WIDTH":
return "Ширина упаковки, см";
case "DIAMETER":
return "Диаметр, см";
default:
return string.Empty;
}
}

private static Dictionary<Region, Dictionary<MeasureType, string>> _region2Measure2Type = new Dictionary<Region, Dictionary<MeasureType, string>>()
{
{
Region.PL, new Dictionary<MeasureType, string>()
{
{ MeasureType.WEIGHT, "WAGA" },
{ MeasureType.WIDTH, "SZEROKOŚĆ" },
{ MeasureType.DIAMETER, "ŚREDNICA" },
{ MeasureType.HEIGHT, "WYSOKOŚĆ" },
{ MeasureType.LENGTH, "DŁUGOŚĆ" }
}
},
{
Region.EN, new Dictionary<MeasureType, string>()
{
{ MeasureType.WEIGHT, "WEIGHT" },
{ MeasureType.WIDTH, "WIDTH" },
{ MeasureType.DIAMETER, "DIAMETER" },
{ MeasureType.HEIGHT, "HEIGHT" },
{ MeasureType.LENGTH, "LENGTH" }
}
},
{
Region.RU, new Dictionary<MeasureType, string>()
{
{ MeasureType.WEIGHT, "ВЕС" },
{ MeasureType.WIDTH, "ШИРИНА" },
{ MeasureType.DIAMETER, "ДИАМЕТР" },
{ MeasureType.HEIGHT, "ВЫСОТА" },
{ MeasureType.LENGTH, "ДЛИННА" }
}
}
};
internal static string PackageSizeNameTranslate(MeasureType mt, Region r)
{
Dictionary<MeasureType, string> measure2LocalName;
if (_region2Measure2Type.TryGetValue(r, out measure2LocalName))
{
string localName;
if (measure2LocalName.TryGetValue(mt, out localName))
{
return localName;
}
}
return string.Empty;
}
}

private Dictionary<string, double> RetailItemCommPackageMeasureParse(JToken root)
{
var measureName2Val = new Dictionary<string, double>();
if (root is JArray)
{
foreach (var pkg in root)
{
var nameOrig = pkg["PackageMeasureType"].Value<string>("$");
var val = pkg["PackageMeasureTextMetric"].Value<string>("$");
val = Regex.Replace(val, @"\D+?$", "");
if (string.IsNullOrEmpty(nameOrig) || string.IsNullOrEmpty(val))
{
continue;
}
measureName2Val[nameOrig] = UtilSmall.ConvertToDouble(val);
}
}
else
{
var nameOrig = root["PackageMeasureType"].Value<string>("$");
var val = root["PackageMeasureTextMetric"].Value<string>("$");
val = Regex.Replace(val, @"\D+?$", "");
if (!string.IsNullOrEmpty(nameOrig) || !string.IsNullOrEmpty(val))
{
measureName2Val[nameOrig] = UtilSmall.ConvertToDouble(val);
}
}
return measureName2Val;
}

private string DescriptionFullBuild(JToken root, string packagesHtml = null, Dictionary<string, string> dimensions = null)
{
var sBuilder = new StringBuilder();
JObject detailes = null;
var benefitsToken = root.SelectToken("$.RetailItemCustomerBenefitList.RetailItemCustomerBenefit");
if (benefitsToken != null && benefitsToken.HasValues)
{
sBuilder.Append("<div><strong>Product details</strong></div>");
if (benefitsToken is JArray)
{
foreach (var b in benefitsToken)
{
sBuilder.AppendFormat("<div>{0}</div>", b["CustomerBenefitText"].Value<string>("$"));
}
}
else
{
sBuilder.AppendFormat("<div>{0}</div>", benefitsToken["CustomerBenefitText"].Value<string>("$"));
}
}
else
{
detailes = root.Value<JObject>("productDescriptionProps");
if (detailes != null)
{
sBuilder.Append("<div><strong>Product details</strong></div>");
var prfs = detailes.Value<JArray>("paragraphs");
if (prfs != null && prfs.HasValues)
{
foreach (var pr in prfs)
{
sBuilder.AppendFormat("<div>{0}</div>", pr.Value<string>());
}
}
var disr = detailes.Value<string>("designerName");
if (!string.IsNullOrEmpty(disr))
{
sBuilder.AppendFormat("<div><strong>Designer</strong></div><div>{0}</div>", disr);
}
}
}

var designerObj = root.SelectToken("$.DesignerNameComm");
if (designerObj != null)
{
sBuilder.AppendFormat("<div><strong>Designer</strong></div><div>{0}</div>", designerObj.Value<string>("$"));
}

if (packagesHtml != null)
{
sBuilder.Append(packagesHtml);
}

JObject accordion = detailes != null ? root.Value<JObject>("accordionObject") : null;
var environmentTokens = root.SelectTokens("$.RetailItemCustomerEnvironmentList..RetailItemEnvironmentText..EnvironmentText");
if (environmentTokens != null)
{
sBuilder.Append("<div><strong>Environment</strong></div>");
foreach (var e in environmentTokens)
{
sBuilder.AppendFormat("<div>{0}</div>", e.Value<string>("$"));
}
}
else if (detailes != null)
{
if (accordion != null)
{
var envData = accordion.SelectToken("sustainabilityAndEnvironment.contentProps.environmentalData");
if (envData != null)
{
var envs = envData as JArray;
if (envs != null && envs.HasValues)
{
sBuilder.Append("<div><strong>Sustainability & environment</strong></div>");
foreach (var e in environmentTokens)
{
var txts = e.Value<JArray>("texts");
if (txts != null && txts.HasValues)
{
foreach (var t in txts)
{
sBuilder.AppendFormat("<div>{0}</div>", t.Value<string>());
}
}

}
}
}
}
}

sBuilder.Append("<div><strong>Materials & care</strong></div>");
var materialsRoot = root.SelectToken("$.RetailItemCustomerMaterialList.RetailItemCustomerMaterial");
if (materialsRoot == null && accordion != null)
{
var m = accordion.SelectToken("materialsAndCare.contentProps.materials");
if (m != null && m.HasValues)
{
materialsRoot = m.First.SelectToken("materials");
}
}
string materialsStr = MaterialStringBuild(materialsRoot);
sBuilder.Append(materialsStr);

private string MaterialStringBuild(JToken materialsRoot)
{
var sBuilder = new StringBuilder();
if (materialsRoot != null && materialsRoot.HasValues)
{
//sBuilder.Append("<div><strong>Материалы</strong></div>");
if (materialsRoot is JArray)
{
foreach (var item in materialsRoot)
{
string materialTokenString = MaterialTokenToString(item);
sBuilder.Append(materialTokenString);
}
}
else
{
string materialTokenString = MaterialTokenToString(materialsRoot);
sBuilder.Append(materialTokenString);
}
}

return sBuilder.ToString();
}

private string MaterialTokenToString(JToken t)
{
var sBuilder = new StringBuilder();
var obj = t as JObject;
if (obj != null && obj.HasValues)
{
var productTypeText = obj["ProductTypeText"];
if (productTypeText != null)
{
sBuilder.AppendFormat("<div>{0}</div>", productTypeText.Value<string>("$"));
}
var retailItemPartMaterial = obj.SelectToken("$..RetailItemPartMaterial");
if (retailItemPartMaterial != null && retailItemPartMaterial.HasValues)
{
if (retailItemPartMaterial is JArray)
{
foreach (var item in retailItemPartMaterial)
{
var materialText = item["MaterialText"];
var partText = item["PartText"];
if (partText != null)
{
sBuilder.AppendFormat("<div>{0} {1}</div>", partText.Value<string>("$"), materialText.Value<string>("$"));
}
else
{
sBuilder.AppendFormat("<div>{0}</div>", materialText.Value<string>("$"));
}
}
}
else
{

private string Translate(string nameOrig)
{
if (nameOrig.Equals("WIDTH"))
{
return "Width";
}
else if (nameOrig.Equals("HEIGHT"))
{
return "Height";
}
else if (nameOrig.Equals("LENGTH"))
{
return "Length";
}
else if (nameOrig.Equals("WEIGHT"))
{
return "Weight";
}
else
{
return nameOrig;
}
}
internal class Variant
{
//public string DataLinkId { get; set; }
//public string DataUrl { get; set; }
public string Id { get; set; }
public string Url { get; set; }
public bool IsSelected { get; set; }
public double Price { get; set; }
}

private void ProductImagesNumberTruncate(Product p, int maxImgsNumber)
{
var imgs = p.ImagesGet();
if (imgs.Count < maxImgsNumber + 1)
{
return;
}
p.ImagesClear();
for (int i = 0; i < maxImgsNumber; i++)
{
p.ImageAdd(imgs.ElementAt(i).Key, imgs.ElementAt(i).Value);
}
}

private Product HtmlBlocksClear(Product p)
{
if (p == null)
{
return null;
}

if (!string.IsNullOrEmpty(p.HtmlBlockProperties))
{
p.HtmlBlockProperties = "";
}
if (!string.IsNullOrEmpty(p.HtmlBlockCombinations))
{
p.HtmlBlockCombinations = "";
}
if (!string.IsNullOrEmpty(p.GetAttributeValue("SYS_IMAGES_ALL")))
{
p.SetAttributeValue("SYS_IMAGES_ALL", "");
}
if (!string.IsNullOrEmpty(p.GetAttributeValue("SYS_ALL_COMB_ATTR")))
{
p.SetAttributeValue("SYS_ALL_COMB_ATTR", "");
}
if (!string.IsNullOrEmpty(p.GetAttributeValue("SYS_COMB_DATA")))
{
p.SetAttributeValue("SYS_COMB_DATA", "");
}
if (!string.IsNullOrEmpty(p.GetAttributeValue("PRICE_FAMILY")))
{
p.SetAttributeValue("PRICE_FAMILY", "");
}
if (!string.IsNullOrEmpty(p.GetAttributeValue("PRODUCT_STYLES")))
{
p.SetAttributeValue("PRODUCT_STYLES", "");
}
if (!string.IsNullOrEmpty(p.GetAttributeValue("VARIATION_SECTION")))
{
p.SetAttributeValue("VARIATION_SECTION", "");
}

return p;
}

private List<Product> ProductsFromCombinationsGet(Product mainProduct)
{
var mods = new List<Product>();
if (mainProduct == null)
{
return mods;
}

foreach (var c in mainProduct.CombinationsGet())
{
var mod = mainProduct.Clone();
mod.Category = mainProduct.Category;
mod.FullDescription = c.FullDescription;
mod.Name = c.Name;
mod.Url = c.Url;
//mod.SetAttributeValue("SYS_PRODUCT_WEIGHT", c.GetAttributeValue("SYS_PRODUCT_WEIGHT"));
mod.CombinationsRemove();
mod.DynamicAttributesClear();
foreach (var attr in c.GetDynamicAttributes())
{
//if (attr.Key.Contains("Вес упаковки"))
//{
// mod.SetAttributeValue("SYS_PRODUCT_WEIGHT", c.GetAttributeValue(attr.Value));
//}
mod.AddDynamicAttribute(attr.Key, c.GetAttributeValue(attr.Value));
}
mod.SetAttributeValue("SYS_PRODUCT_WEIGHT", c.Weight);
mod.SetAttributeValue("SYS_PRODUCT_SKU", c.Art);
mod.SetAttributeValue("SYS_PRODUCT_ART", c.Art);
mod.Quantity = c.Quantity;
mod.Price = c.Price;
mod.PriceOld = c.PriceOld;
mod.ImagesClear();
foreach (var img in c.ImagesGet())
{
mod.ImageAdd(img.Key, img.Value);
}
mods.Add(mod);
}
return mods;
}

private string IDCreate(string sourceID)
{
if (string.IsNullOrEmpty(sourceID))
{
return sourceID;
}

sourceID = Regex.Replace(sourceID, @"\D", "");
var arr = sourceID.ToCharArray();
var newChars = new List<char>();

for (int i = 0; i < arr.Length; i++)
{
if (i > 0 && i % 3 == 0)
{
newChars.Add(‘.’);
}
newChars.Add(arr[i]);
}

string newID = new string(newChars.ToArray());

if (!string.IsNullOrEmpty(newID))
{
newID = newID.TrimEnd(‘.’);
}

return newID;
}

private bool IsDuplicateCategory(Combination cmb, List<int> propsHashes)
{
var attrs = cmb.GetDynamicAttributes();
if (attrs.Count < 1)
{
return false;//return true;
}

var strBuilder = new StringBuilder();
foreach (var attr in attrs)
{
strBuilder.Append(cmb.GetAttributeValue(attr.Value));
}
int hash = strBuilder.ToString()
.GetHashCode();

if (propsHashes.Contains(hash))
{
return true;
}

propsHashes.Add(hash);
return false;
}

private HttpMethodParameters _hmpForQuantity = new HttpMethodParameters()
{
MethodName = "GET",
Accept = "application/json;version=2",
//Accept = "application/vnd.ikea.iows+json;version=2.0",
httpHeaders = new Dictionary<string, string>()
{
/*
* https://www.ikea.com/ru/ru/products/javascripts/range-pip-main.751a00ccf1e16b028440.js
* "x-client-id" прописан хардкодом в скрипте.
* В один прекрасный момент, все может измениться 🙂
*/
{ "X-Client-Id", "b6c117e5-ae61-4ef5-b4cc-e0b1e37f0631" },
{ "accept-encoding", "gzip, deflate, br" },

/*
* https://www.ikea.com/ext/us/task-rabbit/js/script.min.js
{"Consumer", "MAMMUT"},
{"Contract", "37249"},
*/
}
};

private int ProductQuantityGet(string shopCode, string productArt)
{
int qty = -1;
_mti.AddLogFormatted("[ProductQuantityGet] Started. Shop code:'{0}’.", shopCode);
var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty();
if (hpl.Load(string.Format("https://api.ingka.ikea.com/cia/availabilities/ru/us?itemNos={0}&expand=StoresList,Restocks", productArt), false, _hmpForQuantity))
{
try
{
var availObject = JsonConvert.DeserializeObject<ModelsIkea.Availability.AvailabilityObject>(hpl.Content);
var shopAvailObj = availObject.availabilities.FirstOrDefault(av => av.classUnitKey.classUnitCode == shopCode);
if (shopAvailObj == null)
{
_mti.AddLogFormatted("[ProductQuantityGet] Can’t find availability info for shop code:'{0}’.", shopCode);
}
else
{
qty = shopAvailObj.buyingOption.cashCarry.availability != null ? shopAvailObj.buyingOption.cashCarry.availability.quantity : 0;
}
}
catch (Exception ex)
{
_mti.AddLogError("Availability parsing failed.", ex);
throw;
}
}
_mti.AddLogFormatted("[ProductQuantityGet] Finished. Shop code:'{0}’ -> Quantity:{1}.", shopCode, qty.ToString());
return qty;
}

private AvailabilityInfo ProductAvailabilityInfoGet(string shopCode, string productArt, string html = null)
{
AvailabilityInfo avi = new AvailabilityInfo() { ItemNo = productArt };
_mti.AddLogFormatted("[ProductAvailabilityInfoGet] Started. Product Art:'{0}’, Shop code:'{1}’, qty html length:{2}.",
productArt, shopCode, !string.IsNullOrEmpty(html) ? html.Length.ToString() : "0");

if (string.IsNullOrEmpty(html))
{
var hpl = _gps.Proxy.GetHtmlPageLoaderEmpty();
hpl.Load(string.Format(_ikeaSettings.RegionSettings.ProductAvailabilityUrlTemplate, productArt), true, _hmpForQuantity);
html = hpl.Content;
}
//if (hpl.Load(string.Format("https://api.ingka.ikea.com/cia/availabilities/ru/us?itemNos={0}&expand=StoresList,Restocks,SalesLocations", productArt), false, _hmpForQuantity))
try
{
var availObject = JsonConvert.DeserializeObject<ModelsIkea.Availability.AvailabilityObject>(html);
ModelsIkea.Availability.Availability shopAvailObj = null;
ModelsIkea.Availability.AvailabilityDataItem shopAvailData = null;
if (availObject.availabilities != null)
{
shopAvailObj = availObject.availabilities.FirstOrDefault(av => av.classUnitKey.classUnitCode == shopCode);
}
if (shopAvailObj == null)
{
if (availObject.data != null)
{
shopAvailData = availObject.data.FirstOrDefault(d => d.classUnitKey.classUnitCode == shopCode && d.isInCashAndCarryRange.HasValue && d.isInCashAndCarryRange.Value);
if (shopAvailData != null)
{
avi.Qty = shopAvailData.availableStocks.First(s => s._type == "CASHCARRY").quantity;
}
}

}
else
{
avi.Qty = shopAvailObj.buyingOption.cashCarry.availability != null ? shopAvailObj.buyingOption.cashCarry.availability.quantity : 0;
}

if (shopAvailObj == null && shopAvailData == null)
{
_mti.AddLogFormatted("[ProductAvailabilityInfoGet] Can’t find availability info for product art:'{0}’ and shop code:'{1}’.", productArt, shopCode);
}

/*
var salesLocationsObj = availObject.salesLocations.FirstOrDefault(sl => sl.classUnitKey.classUnitCode == shopCode);
if (salesLocationsObj == null)
{
_mti.AddLogFormatted("[ProductAvailabilityInfoGet] Can’t find sales locations info for shop code:'{0}’.", shopCode);
}
else
{
if (salesLocationsObj.childItems != null)
{
var salesLocations = salesLocationsObj.childItems
.Where(ch => ch.salesLocations != null
&& ch.salesLocations.Length > 0);

if (salesLocations.Any())
{
foreach (var ch in salesLocations)
{
if (ch.salesLocations[0].aisleAndBin != null)
{
avi.SalesLocations.Add(new SalesLocationInfo()
{
ItemNo = IDCreate(ch.itemKey.itemNo),
Aisle = ch.salesLocations[0].aisleAndBin.aisle,
Bin = ch.salesLocations[0].aisleAndBin.bin
});
}
}
}
}
else
{
avi.SalesLocations = salesLocationsObj.salesLocations
.Where(s => s != null && s.aisleAndBin != null)
.Select(s => new SalesLocationInfo()
{
ItemNo = IDCreate(salesLocationsObj.itemKey.itemNo),
Aisle = s.aisleAndBin.aisle,
Bin = s.aisleAndBin.bin
}).ToList();
}
}
*/
}
catch (Exception ex)
{
_mti.AddLogError("Availability parsing failed.", ex);
throw;
}
_mti.AddLogFormatted("[ProductAvailabilityInfoGet] Finished. Product Art:'{0}’, Shop code:'{1}’ -> Quantity:{2}, SalesLocations count:{3}.",
productArt, shopCode, avi.Qty.ToString(), avi.SalesLocations.Count.ToString());

return avi;
}
private void ProductImagesRename(Product p)
{
var imgs = p.ImagesGet();
p.ImagesClear();
int idx = 1;
var newImgs = new Dictionary<string, string>();

foreach (var img in imgs)
{
newImgs[img.Key] = MakeImgName2Sku(img.Key, idx, p.Art.Replace(".", ""));//product.ID);
idx++;
}

p.ImageAdd(newImgs);
}

private string MakeImgName2Sku(string imgHref, int idx, string source)
{
var expanImg = new Regex(@"\.(jpg|png|bmp|gif|jpeg)", RegexOptions.IgnoreCase).Match(imgHref).Value;
if (string.IsNullOrEmpty(expanImg))
{
expanImg = ".jpg";
}
if (idx > 1)
{
return source + "-" + idx.ToString(CultureInfo.InvariantCulture) + expanImg;
}
return source + expanImg;
}

private Dictionary<string, double> PackageSizesGet(string html)
{
string pattern = @"var jProductData = (\{[\w\W]*?\});";

if (string.IsNullOrEmpty(html) || !Regex.IsMatch(html, pattern) || string.IsNullOrEmpty(_product.Url))
{
return new Dictionary<string, double>();
}

string json = Regex.Match(html, pattern).Groups[1].Value;
var sizeName2Value = new Dictionary<string, double>();

try
{
string partNumber = new Uri(_product.Url).Segments.Last().TrimEnd(new[] { ‘/’ });
var jObj = JObject.Parse(json);
var mainProdJtoken = jObj["product"]["items"].FirstOrDefault(j => ((string)j["partNumber"]).Equals(partNumber));

if (mainProdJtoken == null)
{
return new Dictionary<string, double>();
}

JArray pkgInfoArr = mainProdJtoken["pkgInfoArr"] as JArray;

if (pkgInfoArr == null)
{
return new Dictionary<string, double>();
}

double maxWidth = 0;
double maxLength = 0;
double packagesQty = 0;
foreach (var pkgInfo in pkgInfoArr.SelectTokens("$..pkgInfo"))
{
if (!_mti.CanContinue())
{
break;
}
foreach (var item in pkgInfo.Children())
{
var qtyProp = item["quantity"];
if (qtyProp == null)
{
continue;
}

int qty = (int)qtyProp;
packagesQty += qty;
foreach (var prop in item.Children<JProperty>())
{
string valueStr = (string)prop.Value;

if (string.IsNullOrEmpty(valueStr))
{
continue;
}

valueStr = Regex.Replace(valueStr, @"(?<=\d)\s.*", "")
.Replace(".", ",");
double value;

if (!double.TryParse(valueStr, out value))
{
continue;
}

string name = prop.Name.Trim();
if (name.Equals("widthMet", StringComparison.InvariantCultureIgnoreCase))
{
maxWidth = (maxWidth > value) ? maxWidth : value;
continue;
}

if (name.Equals("lengthMet", StringComparison.InvariantCultureIgnoreCase))
{
maxLength = (maxLength > value) ? maxLength : value;
continue;
}

if (sizeName2Value.Keys.Contains(prop.Name.Trim()))
{
sizeName2Value[prop.Name.Trim()] += value * qty;
}
else
{
sizeName2Value[prop.Name.Trim()] = value;
}
}
}
}
sizeName2Value["widthMet"] = maxWidth;
sizeName2Value["lengthMet"] = maxLength;
sizeName2Value["packagesQty"] = packagesQty;

for (int i = 0; i < sizeName2Value.Count; i++)
{
sizeName2Value[sizeName2Value.ElementAt(i).Key] = Math.Round(sizeName2Value.ElementAt(i).Value, 2);
}
}
catch (Exception ex)
{
_mti.AddLogError("<<<<< PackageSizesGet() failed.", ex);
}

return sizeName2Value;
}

private Dictionary<string, double> PackageSizesGet(JObject jObj, string partNumber)
{
if (string.IsNullOrEmpty(partNumber) || jObj == null)
{
return new Dictionary<string, double>();
}

var sizeName2Value = new Dictionary<string, double>();

try
{
var mainProdJtoken = jObj["product"]["items"].FirstOrDefault(j => ((string)j["partNumber"]).Equals(partNumber));

if (mainProdJtoken == null)
{
return new Dictionary<string, double>();
}

JArray pkgInfoArr = mainProdJtoken["pkgInfoArr"] as JArray;

if (pkgInfoArr == null)
{
return new Dictionary<string, double>();
}

int qty = (int)qtyProp;
packagesQty += qty;
foreach (var prop in item.Children<JProperty>())
{
string valueStr = (string)prop.Value;

if (string.IsNullOrEmpty(valueStr))
{
continue;
}

valueStr = Regex.Replace(valueStr, @"(?<=\d)\s.*", "")
.Replace(".", ",");
double value;

if (!double.TryParse(valueStr, out value))
{
continue;
}

string name = prop.Name.Trim();
if (name.Equals("widthMet", StringComparison.InvariantCultureIgnoreCase))
{
maxWidth = (maxWidth > value) ? maxWidth : value;
continue;
}

if (name.Equals("lengthMet", StringComparison.InvariantCultureIgnoreCase))
{
maxLength = (maxLength > value) ? maxLength : value;
continue;
}

return sizeName2Value;
}
private void Init(GrabCatalogBuildScriptParameters p)
{
_mti = p.Process.m_ti;
_category = new Category { ID = "0" };
}
}

namespace ModelsIkea
{
namespace Info
{
public class InformationSection
{
public bool isClient { get; set; }
public Dimensionprops dimensionProps { get; set; }
public Productdetailsprops productDetailsProps { get; set; }
public object productReviewsProps { get; set; }
public bool technicalInformationProps { get; set; }
}

public class Dimensionprops
{
public string title { get; set; }
public string subtitle { get; set; }
public Dimension[] dimensions { get; set; }
public object[] images { get; set; }
public Fallbackimage fallbackImage { get; set; }
public string eventLabel { get; set; }
}

public class Fallbackimage
{
public string alt { get; set; }
public string id { get; set; }
public string imageFileName { get; set; }
public string url { get; set; }
public string type { get; set; }
}

public class Dimension
{
public string type { get; set; }
public string name { get; set; }
public string measure { get; set; }
}

public class Productdetailsprops
{
public string title { get; set; }
public string eventLabel { get; set; }
public Productdescriptionprops productDescriptionProps { get; set; }
public Accordionobject accordionObject { get; set; }
}

public class Productdescriptionprops
{
public string[] paragraphs { get; set; }
public string designerLabel { get; set; }
public string designerName { get; set; }
}

public class Accordionobject
{
public Materialsandcare materialsAndCare { get; set; }
public object sustainabilityAndEnvironment { get; set; }
public Packaging packaging { get; set; }
public Assemblyanddocuments assemblyAndDocuments { get; set; }
}

public class Materialsandcare
{
public string id { get; set; }
public string title { get; set; }
public Contentprops contentProps { get; set; }
}

public class Contentprops
{
public Material[] materials { get; set; }
public Careinstruction[] careInstructions { get; set; }
public object[] proposition65 { get; set; }
public Package[] packages { get; set; }
public string totalNoOfPackagesText { get; set; }
public Attachments attachments { get; set; }
}

public class Material
{
public string productType { get; set; }
//public Material1[] materials { get; set; }
public Material[] materials { get; set; }
public string material { get; set; }
public string part { get; set; }
}

//public class Material1
//{
// public string material { get; set; }
// public string part { get; set; }
//}

public class Careinstruction
{
public string[] texts { get; set; }
}

public class Packaging
{
public string id { get; set; }
public string title { get; set; }
//public Contentprops1 contentProps { get; set; }
public Contentprops contentProps { get; set; }

}

//public class Contentprops1
//{
// public Package[] packages { get; set; }
// public string totalNoOfPackagesText { get; set; }
//}

public class Package
{
public string name { get; set; }
public string typeName { get; set; }
public Articlenumber articleNumber { get; set; }
public Measurement[][] measurements { get; set; }
public Quantity quantity { get; set; }
public bool multiPackDisclaimerText { get; set; }
}

public class Articlenumber
{
public string label { get; set; }
public string value { get; set; }
}

public class Quantity
{
public string label { get; set; }
public int value { get; set; }
}

public class Measurement
{
public string label { get; set; }
public string value { get; set; }
}

public class Assemblyanddocuments
{
public string id { get; set; }
public string title { get; set; }
//public Contentprops2 contentProps { get; set; }
public Contentprops contentProps { get; set; }
}

//public class Contentprops2
//{
// public Attachments attachments { get; set; }
//}

public class Attachments
{
public Assembly assembly { get; set; }
public Software software { get; set; }
public Other other { get; set; }
public Disassembly disassembly { get; set; }
}

public class Assembly
{
public string header { get; set; }
public Attachment[] attachments { get; set; }
}

public class Attachment
{
public string id { get; set; }
public string url { get; set; }
public string label { get; set; }
}

public class Software
{
public string header { get; set; }
public object[] attachments { get; set; }
}

public class Other
{
public string header { get; set; }
public object[] attachments { get; set; }
}

public class Disassembly
{
public string header { get; set; }
public object[] attachments { get; set; }
}
}

namespace Price
{

public class PricePackage
{
public object newProductText { get; set; }
public string newLowerPriceText { get; set; }
public object familyText { get; set; }
public object timeRestrictedOfferText { get; set; }
public string productName { get; set; }
public string productDescription { get; set; }
public string measurementText { get; set; }
public Price price { get; set; }
public object productReviewsProps { get; set; }
public object productDiscountsProps { get; set; }
public object ecoWeeeFees { get; set; }
public object timeFrame { get; set; }
public object vatInfoText { get; set; }
public object energyLabelProps { get; set; }
public string ecoWeeeFeeExcludeText { get; set; }
public string totalPriceText { get; set; }
public string ecoWeeeFeeLinkText { get; set; }
public object ecoWeeeFeeLink { get; set; }
public object validToFromText { get; set; }
public object validToFromAndWhileSupplyLastText { get; set; }
public object whileSupplyLastText { get; set; }
}

public class Price
{
public Mainpriceprops mainPriceProps { get; set; }
public Previouspriceprops previousPriceProps { get; set; }
public string previousPriceText { get; set; }
public object comparisonPriceText { get; set; }
public string discount { get; set; }
}

public class Mainpriceprops
{
public Price1 price { get; set; }
public string unit { get; set; }
public string currencySymbol { get; set; }
public string separator { get; set; }
public bool hasTrailingCurrency { get; set; }
public bool hasHighlight { get; set; }
public bool hasStrikeThrough { get; set; }
public bool isSmall { get; set; }
public bool isSecondary { get; set; }
}

public class Price1
{
public string integer { get; set; }
public string decimals { get; set; }
}

public class Previouspriceprops
{
//public Price2 price { get; set; }
public Price1 price { get; set; }
public string unit { get; set; }
public string currencySymbol { get; set; }
public string separator { get; set; }
public bool hasTrailingCurrency { get; set; }
public bool hasHighlight { get; set; }
public bool hasStrikeThrough { get; set; }
public bool isSmall { get; set; }
public bool isSecondary { get; set; }
}

//public class Price2
//{
// public string integer { get; set; }
// public string decimals { get; set; }
//}

}

namespace Variations
{

public class VariationsSection
{
public List<Variation> variations { get; set; }
public bool disabled { get; set; }
public VariationsSection()
{
variations = new List<Variation>();
}
}

public class Variation
{
public string title { get; set; }
public string selectedOption { get; set; }
public List<Option> allOptions { get; set; } // akl_02112021: a new field of variations appeared.
public List<Option> options { get; set; }
public string code { get; set; }
public Variation()
{
options = new List<Option>();
allOptions = new List<Option>();
}
}

public class Option
{
public string url { get; set; }
public string title { get; set; }
public bool isSelected { get; set; }
public string linkId { get; set; }
public Image image { get; set; }
public Priceprops priceProps { get; set; }
}

public class Image
{
public string alt { get; set; }
public string id { get; set; }
public string imageFileName { get; set; }
public string url { get; set; }
public string type { get; set; }
}

public class Priceprops
{
public string priceNumeral { get; set; }
public Revampprice revampPrice { get; set; }
public Priceformat priceFormat { get; set; }
}

public class Revampprice
{
public int numDecimals { get; set; }
public string separator { get; set; }
public string integer { get; set; }
public string decimals { get; set; }
public string currencySymbol { get; set; }
public string currencyPrefix { get; set; }
public string currencySuffix { get; set; }
public bool hasTrailingCurrency { get; set; }

public double PriceDoubleGet()
{
var fmt = new NumberFormatInfo();
fmt.NumberDecimalSeparator = separator;
double r;
if (double.TryParse(string.Format("{0}{1}{2}", integer, separator, decimals), NumberStyles.Any, fmt, out r))
{
return r;
}
return 0;
}
}

public class Priceformat
{
public string currencyPrefix { get; set; }
public string currencySuffix { get; set; }
public string priceFormatter { get; set; }
}

}

//namespace Availability
//{

// public class AvailabilityResponse
// {
// public Stockavailability StockAvailability { get; set; }
// }

// public class Stockavailability
// {
// public Classunitkey ClassUnitKey { get; set; }
// public Itemkey ItemKey { get; set; }
// public Retailitemavailability RetailItemAvailability { get; set; }
// public Availablestockforecastlist AvailableStockForecastList { get; set; }
// public Xmlns xmlns { get; set; }
// }

// public class Classunitkey
// {
// public Classtype ClassType { get; set; }
// public Classunittype ClassUnitType { get; set; }
// public Classunitcode ClassUnitCode { get; set; }
// }

// public class Classtype
// {
// [JsonProperty("$")]
// public string ItemVal { get; set; }
// }

// public class Classunittype
// {
// [JsonProperty("$")]
// public string ItemVal { get; set; }
// }

// public class Classunitcode
// {
// [JsonProperty("$")]
// public string ItemVal { get; set; }
// }

// public class Itemkey
// {
// public Itemno ItemNo { get; set; }
// public Itemtype ItemType { get; set; }
// }

// public class Itemno
// {
// [JsonProperty("$")]
// public string ItemVal { get; set; }
// }

// public class Itemtype
// {
// [JsonProperty("$")]
// public string ItemVal { get; set; }
// }

// public class Retailitemavailability
// {
// public Availablestock AvailableStock { get; set; }
// public Availablestocktype AvailableStockType { get; set; }
// public Instockprobabilitycode InStockProbabilityCode { get; set; }
// public Instockrangecode InStockRangeCode { get; set; }
// public Incustomerorderrangecode InCustomerOrderRangeCode { get; set; }
// public Retailitemcommchildavailabilitylist RetailItemCommChildAvailabilityList { get; set; }
// }

// public class Availablestock
// {
// [JsonProperty("$")]
// public int? ItemVal { get; set; }
// }

// public class Availablestocktype
// {
// [JsonProperty("$")]
// public string ItemVal { get; set; }
// }

// public class Instockprobabilitycode
// {
// [JsonProperty("$")]
// public string ItemVal { get; set; }
// }

// public class Instockrangecode
// {
// [JsonProperty("$")]
// public string ItemVal { get; set; }
// }

// public class Incustomerorderrangecode
// {
// [JsonProperty("$")]
// public string ItemVal { get; set; }
// }

// public class Retailitemcommchildavailabilitylist
// {
// public Retailitemcommchildavailability[] RetailItemCommChildAvailability { get; set; }
// }

// public class Retailitemcommchildavailability
// {
// //public Itemno1 ItemNo { get; set; }
// public Itemno ItemNo { get; set; }
// //public Itemtype1 ItemType { get; set; }
// public Itemtype ItemType { get; set; }
// //public Availablestocktype1 AvailableStockType { get; set; }
// public Availablestocktype AvailableStockType { get; set; }
// public Recommendedsaleslocation RecommendedSalesLocation { get; set; }
// public Salesmethodcode SalesMethodCode { get; set; }
// }

// //public class Itemno1
// //{
// // public object _ { get; set; }
// //}

// //public class Itemtype1
// //{
// // public string _ { get; set; }
// //}

// //public class Availablestocktype1
// //{
// // public string _ { get; set; }
// //}

// public class Recommendedsaleslocation
// {
// }

// public class Salesmethodcode
// {
// [JsonProperty("$")]
// public string ItemVal { get; set; }
// }

// public class Availablestockforecastlist
// {
// public Availablestockforecast[] AvailableStockForecast { get; set; }
// }

// public class Availablestockforecast
// {
// //public Availablestock1 AvailableStock { get; set; }
// public Availablestock AvailableStock { get; set; }
// //public Availablestocktype2 AvailableStockType { get; set; }
// public Availablestocktype AvailableStockType { get; set; }
// //public Instockprobabilitycode1 InStockProbabilityCode { get; set; }
// public Instockprobabilitycode InStockProbabilityCode { get; set; }
// public Validdatetime ValidDateTime { get; set; }
// public Validdatetimeunit ValidDateTimeUnit { get; set; }
// }

// //public class Availablestock1
// //{
// // public int _ { get; set; }
// //}

// //public class Availablestocktype2
// //{
// // public string _ { get; set; }
// //}

// //public class Instockprobabilitycode1
// //{
// // public string _ { get; set; }
// //}

// public class Validdatetime
// {
// [JsonProperty("$")]
// public string ItemVal { get; set; }
// }

// public class Validdatetimeunit
// {
// [JsonProperty("$")]
// public string ItemVal { get; set; }
// }

// public class Xmlns
// {
// [JsonProperty("$")]
// public string ItemVal { get; set; }
// }

//}

namespace Availability
{
public class AvailabilityObject
{
public Availability[] availabilities { get; set; }
public SalesLocation[] salesLocations { get; set; }
public AvailabilityDataItem[] data { get; set; }
public DateTime timestamp { get; set; }
public string traceId { get; set; }
}

public class AvailabilityDataItem
{
public bool? isInCashAndCarryRange { get; set; }
public bool isInHomeDeliveryRange { get; set; }
public AvailableStock[] availableStocks { get; set; }
public Classunitkey classUnitKey { get; set; }
public Itemkey itemKey { get; set; }
}

public class AvailableStock
{
[JsonProperty("type")]
public string _type { get; set; }
public int quantity { get; set; }
}

public class SalesLocation
{
public Childitem[] childItems { get; set; }
public Classunitkey classUnitKey { get; set; }
public Itemkey itemKey { get; set; }
public Saleslocation[] salesLocations { get; set; }
}

//public class Classunitkey
//{
// public string classUnitCode { get; set; }
// public string classUnitType { get; set; }
//}

//public class Itemkey
//{
// public string itemNo { get; set; }
// public string itemType { get; set; }
//}

public class Childitem
{
public Itemkey itemKey { get; set; }
public Saleslocation[] salesLocations { get; set; }
}

//public class Itemkey1
//{
// public string itemNo { get; set; }
// public string itemType { get; set; }
//}

public class Saleslocation
{
public Aisleandbin aisleAndBin { get; set; }
public string division { get; set; }
public string id { get; set; }
public int recommendationRank { get; set; }
public string type { get; set; }
}

public class Aisleandbin
{
public string aisle { get; set; }
public string bin { get; set; }
}

public class Availability
{
public bool availableForCashCarry { get; set; }
public bool availableForClickCollect { get; set; }
public bool availableForHomeDelivery { get; set; }
public Buyingoption buyingOption { get; set; }
public Classunitkey classUnitKey { get; set; }
public Itemkey itemKey { get; set; }
}

public class Buyingoption
{
public Cashcarry cashCarry { get; set; }
public Homedelivery homeDelivery { get; set; }
}

public class Cashcarry
{
public Availability1 availability { get; set; }
public Range range { get; set; }
public string unitOfMeasure { get; set; }
}

public class Availability1
{
public Probability probability { get; set; }
public int quantity { get; set; }
public DateTime updateDateTime { get; set; }
public Restock[] restocks { get; set; }
}

public class Probability
{
public Thisday thisDay { get; set; }
public DateTime updateDateTime { get; set; }
}

public class Thisday
{
public Colour colour { get; set; }
public string messageType { get; set; }
}

public class Colour
{
public string rgbDec { get; set; }
public string rgbHex { get; set; }
}

public class Restock
{
public string earliestDate { get; set; }
public string latestDate { get; set; }
public int quantity { get; set; }
public string reliability { get; set; }
public string type { get; set; }
public DateTime updateDateTime { get; set; }
}

public class Range
{
public bool inRange { get; set; }
}

public class Homedelivery
{
public Range1 range { get; set; }
public Availability2 availability { get; set; }
}

public class Range1
{
public bool inRange { get; set; }
}

public class Availability2
{
public Probability1 probability { get; set; }
public DateTime updateDateTime { get; set; }
}

public class Probability1
{
public Thisday1 thisDay { get; set; }
public DateTime updateDateTime { get; set; }
}

public class Thisday1
{
public Colour1 colour { get; set; }
public string messageType { get; set; }
}

public class Colour1
{
public string rgbDec { get; set; }
public string rgbHex { get; set; }
}

public class Classunitkey
{
public string classUnitCode { get; set; }
public string classUnitType { get; set; }
}

public class Itemkey
{
public string itemNo { get; set; }
public string itemType { get; set; }
}

}

namespace Cataloge
{

public class Category
{
public string id { get; set; }
public int depth { get; set; }
public List<Subcategory> subcategories { get; set; }
public Category()
{
subcategories = new List<Subcategory>();
}
}

public class Subcategory
{
public string id { get; set; }
public string link { get; set; }
public string title { get; set; }
public bool isCrossReferenced { get; set; }
}

}

namespace Media
{
public class MediaListBase
{
public string type { get; set; }
public Content content { get; set; }
}
public class MediaGrid
{
public List<Fullmedialist> fullMediaList { get; set; }
public List<Medialist> mediaList { get; set; }
public int heroesCount { get; set; }
public bool isDoubleHero { get; set; }
public bool isSingleHero { get; set; }
public bool disabledButton { get; set; }
public string buttonMoreLabel { get; set; }
public string buttonLessLabel { get; set; }
public int barPosition { get; set; }
public float barWidth { get; set; }
public bool isSingleMedia { get; set; }
public object energyLabelProps { get; set; }
public MediaGrid()
{
fullMediaList = new List<Fullmedialist>();
mediaList = new List<Medialist>();
}
}

public class Fullmedialist : MediaListBase
{
//public string type { get; set; }
//public Content content { get; set; }
}

public class Content
{
public string alt { get; set; }
public string id { get; set; }
public string imageFileName { get; set; }
public string url { get; set; }
public string type { get; set; }
}

public class Medialist : MediaListBase
{
//public string type { get; set; }
//public Content content { get; set; }
}
}

namespace Pagination
{

public class DataCategory
{
public string id { get; set; }
public Pagination pagination { get; set; }
public int totalCount { get; set; }
}

public class Pagination
{
public int currentPage { get; set; }
public Next next { get; set; }
public object prev { get; set; }
}

public class Next
{
public string url { get; set; }
public string label { get; set; }
}

//public class MoreproductsResponse
//{
// public string usergroup { get; set; }
// public Moreproducts moreProducts { get; set; }
//}

//public class Moreproducts
//{
// public List<Productwindow> productWindow { get; set; }
// public Moreproducts()
// {
// productWindow = new List<Productwindow>();
// }
//}

//public class Productwindow
//{
// public string name { get; set; }
// public string typeName { get; set; }
// public string itemMeasureReferenceText { get; set; }
// public string mainImageUrl { get; set; }
// public string pipUrl { get; set; }
// public string id { get; set; }
// public string metadata { get; set; }
// public bool onlineSellable { get; set; }
// public Gprdescription gprDescription { get; set; }
// public Color[] colors { get; set; }
// public float priceNumeral { get; set; }
// public string currencyCode { get; set; }
//}

//public class Gprdescription
//{
// public object[] colors { get; set; }
// public int numberOfSizes { get; set; }
// public int numberOfColors { get; set; }
//}

//public class Color
//{
// public string name { get; set; }
// public string id { get; set; }
//}

public class MoreproductsResponse
{
public string usergroup { get; set; }
public Moreproducts moreProducts { get; set; }
}

public class Moreproducts
{
public List<Productwindow> productWindow { get; set; }
public Moreproducts()
{
productWindow = new List<Productwindow>();
}
}

public class Productwindow
{
public string name { get; set; }
public string typeName { get; set; }
public string itemMeasureReferenceText { get; set; }
public string mainImageUrl { get; set; }
public string pipUrl { get; set; }
public string id { get; set; }
public string itemNoGlobal { get; set; }
public string metadata { get; set; }
public bool onlineSellable { get; set; }
public bool lastChance { get; set; }
public Gprdescription gprDescription { get; set; }
public Color[] colors { get; set; }
public double priceNumeral { get; set; }
public string currencyCode { get; set; }
}

public class Gprdescription
{
public int numberOfVariants { get; set; }
public List<Variant> variants { get; set; }
public Gprdescription()
{
variants = new List<Variant>();
}
}

public class Variant
{
public string id { get; set; }
public string pipUrl { get; set; }
public string imageUrl { get; set; }
public string imageAlt { get; set; }
//public Price price { get; set; }
}
public class Price
{
public string prefix { get; set; }
public string wholeNumber { get; set; }
public string separator { get; set; }
public string decimals { get; set; }
public string suffix { get; set; }
public bool isRegularCurrency { get; set; }
}

public class Color
{
public string name { get; set; }
public string id { get; set; }
public string hex { get; set; }
}
}
namespace ProductProps
{
/*
* akl_02152021:
* Теперь информация об упаковках приходит внутри html страницы товара.
*/

public class GeneralDataProps
{
public string productName { get; set; }
public bool hideSection { get; set; }
public string productNumber { get; set; }
public string productType { get; set; }
public bool onlineSellable { get; set; }
public bool enabledCartButton { get; set; }
public string noJavascriptLabel { get; set; }
public string currency { get; set; }
public string productPrice { get; set; }
public string ratings { get; set; }
}

public class Props
{
public bool isClient { get; set; }
public JToken dimensionProps { get; set; }
public Dimensionprops DimensionPropsObject
{
get
{
Dimensionprops dp = null;
try
{
var jo = dimensionProps as JObject;
if (jo != null)
{
dp = jo.ToObject<Dimensionprops>();
}
}
catch { }
return dp;
}
}
public Productdetailsprops productDetailsProps { get; set; }
// public object productReviewsProps { get; set; }
//public bool technicalInformationProps { get; set; }
}

public class Dimensionprops
{
public string title { get; set; }
public Packaging packaging { get; set; }
//public Dimension[] dimensions { get; set; }
//public object[] images { get; set; }
//public Fallbackimage fallbackImage { get; set; }
//public string eventLabel { get; set; }
}

public class Dimension
{
public string type { get; set; }
public string name { get; set; }
public string measure { get; set; }
}

public class Productdescriptionprops
{
public string[] paragraphs { get; set; }
public string designerLabel { get; set; }
public string designerName { get; set; }
}

public class Materialsandcare
{
public string id { get; set; }
public string title { get; set; }
public Contentprops contentProps { get; set; }
}

public class Contentprops
{
public Material[] materials { get; set; }
public object[] careInstructions { get; set; }
public object[] proposition65 { get; set; }
}

public class Material
{
public string productType { get; set; }
public Material1[] materials { get; set; }
}

public class Material1
{
public string part { get; set; }
public string material { get; set; }
}

public class Packaging
{
public string id { get; set; }
public string title { get; set; }
public Contentprops1 contentProps { get; set; }
}

public class Contentprops1
{
public List<Package> packages { get; set; }
public string totalNoOfPackagesText { get; set; }
public Contentprops1()
{
packages = new List<Package>();
}
}

public class Articlenumber
{
public string label { get; set; }
public string value { get; set; }
}

public class Quantity
{
public string label { get; set; }
public int value { get; set; }
}

public class Measurement
{
public string label { get; set; }
public string value { get; set; }
}

public class Assemblyanddocuments
{
public string id { get; set; }
public string title { get; set; }
public Contentprops2 contentProps { get; set; }
}

public class Contentprops2
{
public Attachments attachments { get; set; }
}

public class Attachments
{
public Assembly assembly { get; set; }
public Software software { get; set; }
public Other other { get; set; }
public Disassembly disassembly { get; set; }
}

public class Assembly
{
public string header { get; set; }
public Attachment[] attachments { get; set; }
}

public class Attachment
{
public string id { get; set; }
public string url { get; set; }
public string label { get; set; }
}

public class Software
{
public string header { get; set; }
public object[] attachments { get; set; }
}

public class Other
{
public string header { get; set; }
public object[] attachments { get; set; }
}

public class Disassembly
{
public string header { get; set; }
public object[] attachments { get; set; }
}

}
}
}
[/code]

Ikea.com country universal scraper – c# code sample

ECommerce Web scrapers