Data Format
Data for how to scrape is kept in a JSON file. Many of the data structures have both verbose JSON definitions, as well as more human readable shorthand structures. For many variables where you can enter an array of objects, you can replace it with a single object if you were only going to have an array of one.
The Base
We start with an empty object
{
"version: 1,
"key": SET_TO_VALUE,
"key2": SET_TO_VALUE,
...
}
Version Number
A version number is required. This let's the parser know which version you are using. If following this documentation always user the integer 1.
Key / Values
This object contains keys / value pairs. The basic keys simple set a value (see Setting a Value) of one of the following:
- config.storeDomain
- config.trackingInterval
- config.currencyDecimalDefault
- config.currencyDecimalWidth
General
- default.currency
- page.type
- page.currency
Category Page
- page.category.id
- page.category.name
Product Page
- page.product.name
- page.product.id
- page.product.sku
- page.product.quantity
- page.product.price
- page.product.image
- page.product.url
- cart.action
- cart.gross
- cart.discount
- cart.discountCode
- cart.action
- order.id
- order.number
- order.gross
- order.discount
- order.discountCode
The advanced keys that include waiting for page changes and looping over data include:
See Advanced Keys for details.
Flat vs Hierarchical
Keys can be either flat or hierarchical. For example:
{
"page.type": SET_TO_VALUE,
"page.product.id": SET_TO_VALUE,
"page.product.name": SET_TO_VALUE,
"cart.loop-item.product.id": SET_TO_VALUE
}
or
{
"page": {
"type": SET_TO_VALUE,
"product": {
"id": SET_TO_VALUE,
"name": SET_TO_VALUE
}
},
"cart": {
"loop-item": {
"product": {
"id": SET_TO_VALUE
}
}
}
}
Example
{
"version": 1,
"default.currency": {
"to": {
"type": "const",
"value": "USD"
}
},
"config": {
"trackingInterval": 250,
"accountKey": "",
"sessionOfferPrefix": "fp_",
"currencyDecimalDefault": "."
},
"page.type": [
{
"to": {
"type": "const",
"value": "home"
},
"if": {
"lookAt": "location.pathname | lowercase",
"tests": "$value == '/'"
}
},
{
"to": {
"type": "const",
"value": "cart"
},
"if": {
"lookAt": "location.pathname | lowercase",
"tests": "$value == '/store/cart/'"
}
},
{
"to": "prod",
"if": {
"reverseLogic": true,
"ifs": [
[
{
"lookAt": {
"type": "var",
"value": "location.pathname",
"filters": [
{
"type": "match",
"params": {
"match": "/([^/]+)/$"
}
}
]
},
"tests": "$value != empty"
}
],
[
{
"lookAt": {
"type": "selector",
"value": "#page_patern #container #middle #home .home_container .headline",
"filters": "text,trim"
},
"tests": "$value != empty"
}
],
[
{
"lookAt": {
"type": "selector",
"value": " .sale",
"filters": "text,currency"
},
"tests": "$value != empty"
},
{
"lookAt": {
"type": "selector",
"value": " .ts-price",
"filters": "text,currency"
},
"tests": "$value != empty"
}
]
]
}
},
{
"to": "cat",
"if": {
"lookAt": "location.pathname | lowercase",
"tests": "$value.indexOf('/category/') > -1"
}
},
{
"to": "order",
"if": {
"lookAt": "location.pathname | lowercase",
"tests": "$value.indexOf('/store/thanks/') = 0"
}
}
],
"page.category.id": [
{
"to": {
"type": "var",
"value": "location.pathname",
"filters": [
{
"type": "match",
"params": {
"match": "/([^/]+)/$"
}
}
]
}
}
],
"page.category.name": [
{
"to": {
"type": "selector",
"value": ".headline",
"filters": [
"text",
"trim"
]
}
}
],
"page.product.id": [
{
"to": {
"type": "var",
"value": "location.pathname",
"filters": [
{
"type": "match",
"params": {
"match": "/([^/]+)/$"
}
}
]
}
}
],
"page.product.sku": [
{
"to": {
"type": "var",
"value": "location.pathname",
"filters": [
{
"type": "match",
"params": {
"match": "/([^/]+)/$"
}
}
]
}
}
],
"page.product.name": [
{
"to": {
"type": "selector",
"value": "#page_patern #container #middle #home .home_container .headline",
"filters": "text,trim"
}
}
],
"page.product.price": [
{
"to": {
"type": "selector",
"value": " .sale",
"filters": "text,currency"
}
},
{
"to": {
"type": "selector",
"value": " .ts-price",
"filters": "text,currency"
}
}
],
"page.product.image": [
{
"to": {
"type": "selector",
"value": "#page_patern .gallery IMG:eq(1)",
"filters": "src"
}
},
{
"to": {
"type": "selector",
"value": "#page_patern .gallery IMG:last",
"filters": "src"
}
}
],
"page.product.url": [
{
"to": {
"type": "var",
"value": "location.href"
}
}
],
"cart.action": [
{
"to": {
"type": "const",
"value": "override"
},
"if": {
"lookAt": "state.page.type",
"tests": "$value == 'cart'"
}
},
{
"to": {
"type": "const",
"value": "repeat"
}
}
],
"cart.loop": {
"type": "selector",
"value": "#cart .cart TBODY TR:has(.remove)"
},
"cart.loop-item.product.name": {
"to": {
"type": "selector",
"value": "TD A",
"filters": "text,trim"
}
},
"cart.loop-item.product.quantity": [
{
"to": {
"type": "selector",
"value": "INPUT",
"filters": "value,number"
}
},
{
"to": {
"type": "selector",
"value": "td:eq(1)",
"filters": "text,number"
}
}
],
"cart.loop-item.product.price": {
"to": {
"type": "selector",
"value": ".money:first",
"filters": [
"text",
"currency"
]
}
},
"cart.loop-item.product.url": {
"to": {
"type": "selector",
"value": "TD A",
"filters": "href"
}
},
"cart.loop-item.product.id": {
"to": {
"type": "selector",
"value": "TD A",
"filters": [
"href",
{
"type": "match",
"params": {
"match": "/([^/]+)/$"
}
}
]
}
},
"cart.loop-item.product.sku": {
"to": {
"type": "var",
"value": "$self.id"
}
},
"cart.gross": {
"to": {
"type": "selector",
"value": ".cart-subtotal",
"filters": "text,currency"
}
},
"cart.discountCode": {
"to": {
"type": "var",
"value": "window.discount_code"
}
},
"cart.discount": {
"to": {
"type": "selector",
"value": ".cart-discount",
"filters": "text,currency"
}
},
"order.id": {
"to": {
"type": "selector",
"value": ".transaction td:first",
"filters": "text,trim"
}
},
"order.number": {
"to": {
"type": "var",
"value": "state.order.id"
}
}
}