Input format sample
Code Block | ||
---|---|---|
| ||
Date,Device,SrcIP,dstIP,Alert,Country SCANNER_1,2012-01-01T13:43:00,10.0.0.1,66.66.66.66,DUMMY_ALERT_TYPE_1,United States SCANNER_2,2012-02-01T14:21:00,SCANNER_2,10.0.0.2,66.66.66.66,DUMMY_ALERT_TYPE_2,United Kingdom SCANNER_3,2012-03-01T15:17:00,10.0.0.1,99.66.99.66,DUMMY_ALERT_TYPE_3,Netherlands |
Source #1 - fileshare
Code Block | ||
---|---|---|
| ||
{ "description": "For cyber demo", "extractType": "Feed", "file": { "XmlRootLevelValues": [], "domain": "DOMAIN", "password": "PASSWORD", "type": "csv", "username": "USER" }, "isPublic": false, "mediaType": "Log", "searchCycle_secs": 3600, "searchIndexFilter": { "metadataFieldList": "" }, "structuredAnalysis": { "associations": [ { "entity1": "$metadata.info.dstIP", "entity2": "$metadata.info.srcIP", "geo_index": "$SCRIPT( return _doc.metadata.info[0].country + '/country'; )", "time_start": "$SCRIPT( return _doc.metadata.info[0].date; )", "verb": "$SCRIPT( return _doc.metadata.info[0].alert; )", "verb_category": "$SCRIPT( return _doc.metadata.info[0].alert; )" } ], "entities": [ { "dimension": "What", "disambiguated_name": "$metadata.info.srcIP", "type": "PrivateIP" }, { "dimension": "What", "disambiguated_name": "$metadata.info.dstIP", "geotag": { "country": "$SCRIPT( return _doc.metadata.info[0].country; )" }, "ontology_type": "country", "type": "PublicIP" }, { "actual_name": "$metadata.info.country", "dimension": "Where", "disambiguated_name": "$SCRIPT( return _doc.metadata.info[0].country; )", "geotag": { "country": "$SCRIPT( return _doc.metadata.info[0].country; )" }, "ontology_type": "country", "type": "Country" }, { "dimension": "What", "disambiguated_name": "$metadata.info.device", "type": "Sensor" }, { "dimension": "What", "disambiguated_name": "$metadata.info.alert", "type": "AlertType" } ], "publishedDate": "$SCRIPT( return _doc.metadata.info[0].date; )", "script": "", "scriptEngine": "javascript", "title": "$metadata.info.alert @ $metadata.info.date [$metadata.info.device]: $metadata.info.dstIP -> $metadata.info.srcIP" }, "tags": [ "cyber", "structured" ], "title": "Cyber Logs Test", "unstructuredAnalysis": { "meta": [ { "context": "First", "fieldName": "info", "script": "var info = decode(text); info;", "scriptlang": "javascript" } ], "script": "function decode(x)\n{\n var info = {}; \n var rec = x.split(','); \n info.device = rec[0];\n info.date = rec[1];\n info.srcIP = rec[2];\n info.dstIP = rec[3];\n info.alert = rec[4];\n info.country = rec[5];\n return info;\n}", "simpleTextCleanser": [ { "field": "fullText", "flags": "md", "replacement": " , ", "script": ",", "scriptlang": "regex" }, { "field": "description", "flags": "md", "replacement": " , ", "script": ",", "scriptlang": "regex" } ] }, "useExtractor":"none", "useTextExtractor":"none", "url": "smb://FILESHARE:139/cyber_logs/" } |
...
Code Block | ||
---|---|---|
| ||
{ "description": "For cyber demo", "extractType": "File", "isPublic": false, "mediaType": "Log", "searchCycle_secs": 3600, "rss": { "extraUrls": [ { "url": "http://INFINITE_ENDPOINT/api/share/get/51ad28a440b4a4f0f757824c?infinite_api_key=API_KEY" } ], "searchConfig": {"script": "var retVals = [];\nvar n = -1;\nvar url = _doc.url.replace(/[?].*/,\"\");\nvar start = 0;\nwhile (start < text.length) {\n var end = text.indexOf('\\n', start);\n if (end == -1) end = text.length;\n var line = text.substr(start,end-1);\n start = end + 1; \n \n n++;\n if (0 == n) continue;\n \n var title = 'line #' + n.toString();\n var url2 = url + '#' + n.toString();\n var fullText = line;\n var retVal = { 'title':title, 'url':url2, 'fullText':line };\n retVals.push(retVal); \n}\nretVals;\n "} }, "searchIndexFilter": { "metadataFieldList": "" }, "structuredAnalysis": { "associations": [ { "entity1": "$metadata.info.dstIP", "entity2": "$metadata.info.srcIP", "geo_index": "$SCRIPT( return _doc.metadata.info[0].country + '/country'; )", "time_start": "$SCRIPT( return _doc.metadata.info[0].date; )", "verb": "$SCRIPT( return _doc.metadata.info[0].alert; )", "verb_category": "$SCRIPT( return _doc.metadata.info[0].alert; )" } ], "entities": [ { "dimension": "What", "disambiguated_name": "$metadata.info.srcIP", "type": "PrivateIP" }, { "dimension": "What", "disambiguated_name": "$metadata.info.dstIP", "geotag": { "country": "$SCRIPT( return _doc.metadata.info[0].country; )" }, "ontology_type": "country", "type": "PublicIP" }, { "actual_name": "$metadata.info.country", "dimension": "Where", "disambiguated_name": "$SCRIPT( return _doc.metadata.info[0].country; )", "geotag": { "country": "$SCRIPT( return _doc.metadata.info[0].country; )" }, "ontology_type": "country", "type": "Country" }, { "dimension": "What", "disambiguated_name": "$metadata.info.device", "type": "Sensor" }, { "dimension": "What", "disambiguated_name": "$metadata.info.alert", "type": "AlertType" } ], "publishedDate": "$SCRIPT( return _doc.metadata.info[0].date; )", "script": "", "scriptEngine": "javascript", "title": "$metadata.info.alert @ $metadata.info.date [$metadata.info.device]: $metadata.info.dstIP -> $metadata.info.srcIP" }, "tags": [ "cyber", "structured" ], "title": "Cyber Logs Test", "unstructuredAnalysis": { "meta": [ { "context": "First", "fieldName": "info", "script": "var info = decode(text); info;", "scriptlang": "javascript" } ], "script": "function decode(x)\n{\n var info = {}; \n var rec = x.split(','); \n info.device = rec[0];\n info.date = rec[1];\n info.srcIP = rec[2];\n info.dstIP = rec[3];\n info.alert = rec[4];\n info.country = rec[5];\n return info;\n}", "simpleTextCleanser": [ { "field": "fullText", "flags": "md", "replacement": " , ", "script": ",", "scriptlang": "regex" }, { "field": "description", "flags": "md", "replacement": " , ", "script": ",", "scriptlang": "regex" } ] }, "useExtractor":"none", "useTextExtractor":"none", "url": "http://INFINITE_ENDPOINT/api/share/get/51ad28a440b4a4f0f757824c?infinite_api_key=API_KEY" } |
Note that the API key is not visible in any of the extracted documents (it is removed in the "searchConfig.script" code), and is also not visible in the source to anyone but the source owner and administrator (due to the "isPublic:false" field). As an alternative (from June 2013), a cookie can be used: (eg) "rss.cookies": "infinitecookie=api:API_KEY;".
...