Page Comparison

 },
        {
            "display": "A global space to group all the complex parsing and processing logic, can be called from anywhere",
            "globals": {
                "scriptlang": "javascript",
                "scripts": ["function create_links( urls, input_array )\n{\n    for (var x in input_array) {\n        var input = input_array[x];\n        urls.push( { url: input.url, title: input.title, description: input.desc, publishedData: input.date, fullText: input.text });\n    }\n}"]
            }
        },
        {
            "display": "Only check the API every 10 minutes (can be set to whatever you'd like)",
            "harvest": {
                "duplicateExistingUrls": true,
                "searchCycle_secs": 600
            }
        },
        {
            "contentMetadata": [{
                "fieldName": "json",
                "index": false,
                "script": "var json = eval('('+text+')'); json; ",
                "scriptlang": "javascript",
                "store": true
            }],
            "display": "Convert the text into a JSON object in the document's metadata field: _doc.metadata.json[0]"
        },
		        {
            "display": "Take the original documents, split them using their metadaata into new documents, and then delete the originals",
            "splitter": {
                "deleteExisting": true,
                "script": "var urls = []; create_links( urls, _metadata.json[0].data ); urls;",
                "scriptflags": "m",
                "scriptlang": "javascript"
            }
        },