Input format sample
Code Block | ||
---|---|---|
| ||
Date,Device,SrcIP,dstIP,Alert,Country SCANNER_1,2012-01-01T13:43:00,10.0.0.1,66.66.66.66,DUMMY_ALERT_TYPE_1,United States SCANNER_2,2012-02-01T14:21:00,SCANNER_2,10.0.0.2,66.66.66.66,DUMMY_ALERT_TYPE_2,United Kingdom SCANNER_3,2012-03-01T15:17:00,10.0.0.1,99.66.99.66,DUMMY_ALERT_TYPE_3,Netherlands |
Source #1 - fileshare
Code Block | ||
---|---|---|
| ||
{ "description": "For cyber demo", "extractType": "Feed", "file": { "XmlRootLevelValues": [], "domain": "DOMAIN", "password": "PASSWORD", "type": "csv", "username": "USER" }, "isPublic": false, "mediaType": "Log", "searchCycle_secs": 3600, "searchIndexFilter": { "metadataFieldList": "" }, "structuredAnalysis": { "associations": [ { "entity1": "$metadata.info.dstIP", "entity2": "$metadata.info.srcIP", "geo_index": "$SCRIPT( return _doc.metadata.info[0].country + '/country'; )", "time_start": "$SCRIPT( return _doc.metadata.info[0].date; )", "verb": "$SCRIPT( return _doc.metadata.info[0].alert; )", "verb_category": "$SCRIPT( return _doc.metadata.info[0].alert; )" } ], "entities": [ { "dimension": "What", "disambiguated_name": "$metadata.info.srcIP", "type": "PrivateIP" }, { "dimension": "What", "disambiguated_name": "$metadata.info.dstIP", "geotag": { "country": "$SCRIPT( return _doc.metadata.info[0].country; )" }, "ontology_type": "country", "type": "PublicIP" }, { "actual_name": "$metadata.info.country", "dimension": "Where", "disambiguated_name": "$SCRIPT( return _doc.metadata.info[0].country; )", "geotag": { "country": "$SCRIPT( return _doc.metadata.info[0].country; )" }, "ontology_type": "country", "type": "Country" }, { "dimension": "What", "disambiguated_name": "$metadata.info.device", "type": "Sensor" }, { "dimension": "What", "disambiguated_name": "$metadata.info.alert", "type": "AlertType" } ], "publishedDate": "$SCRIPT( return _doc.metadata.info[0].date; )", "script": "", "scriptEngine": "javascript", "title": "$metadata.info.alert @ $metadata.info.date [$metadata.info.device]: $metadata.info.dstIP -> $metadata.info.srcIP" }, "tags": [ "cyber", "structured" ], "title": "Cyber Logs Test", "unstructuredAnalysis": { "meta": [ { "context": "First", "fieldName": "info", "script": "var info = decode(text); info;", "scriptlang": "javascript" } ], "script": "function decode(x)\n{\n var info = {}; \n var rec = x.split(','); \n info.device = rec[0];\n info.date = rec[1];\n info.srcIP = rec[2];\n info.dstIP = rec[3];\n info.alert = rec[4];\n info.country = rec[5];\n return info;\n}", "simpleTextCleanser": [ { "field": "fullText", "flags": "md", "replacement": " , ", "script": ",", "scriptlang": "regex" }, { "field": "description", "flags": "md", "replacement": " , ", "script": ",", "scriptlang": "regex" } ] }, "useExtractor":"none", "useTextExtractor":"none", "url": "smb://FILESHARE:139/cyber_logs/" } |
...
It is slightly more complicated to parse CSV files over the Web, but still quite possible, using the searchConfig capability. Note that one neat trick is to upload a share to Infinit.e, and then use an API key to to access the REST interface. Users can allocate themselves an API key from the People Manager.
Code Block | ||
---|---|---|
| ||
{ "description": "For cyber demo", "extractType": "File", "isPublic": false, "mediaType": "Log", "searchCycle_secs": 3600, "rss": { "extraUrls": [ { "url": "http://INFINITE_ENDPOINT/api/share/get/51ad28a440b4a4f0f757824c?infinite_api_key=API_KEY" } ], "searchConfig": {"script": "var retVals = [];\nvar n = -1;\nvar url = _doc.url.replace(/[?].*/,\"\");\nvar start = 0;\nwhile (start < text.length) {\n var end = text.indexOf('\\n', start);\n if (end == -1) end = text.length;\n var line = text.substr(start,end-1);\n start = end + 1; \n \n n++;\n if (0 == n) continue;\n \n var title = 'line #' + n.toString();\n var url2 = url + '#' + n.toString();\n var fullText = line;\n var retVal = { 'title':title, 'url':url2, 'fullText':line };\n retVals.push(retVal); \n}\nretVals;\n "} }, "searchIndexFilter": { "metadataFieldList": "" }, "structuredAnalysis": { "associations": [ { "entity1": "$metadata.info.dstIP", "entity2": "$metadata.info.srcIP", "geo_index": "$SCRIPT( return _doc.metadata.info[0].country + '/country'; )", "time_start": "$SCRIPT( return _doc.metadata.info[0].date; )", "verb": "$SCRIPT( return _doc.metadata.info[0].alert; )", "verb_category": "$SCRIPT( return _doc.metadata.info[0].alert; )" } ], "entities": [ { "dimension": "What", "disambiguated_name": "$metadata.info.srcIP", "type": "PrivateIP" }, { "dimension": "What", "disambiguated_name": "$metadata.info.dstIP", "geotag": { "country": "$SCRIPT( return _doc.metadata.info[0].country; )" }, "ontology_type": "country", "type": "PublicIP" }, { "actual_name": "$metadata.info.country", "dimension": "Where", "disambiguated_name": "$SCRIPT( return _doc.metadata.info[0].country; )", "geotag": { "country": "$SCRIPT( return _doc.metadata.info[0].country; )" }, "ontology_type": "country", "type": "Country" }, { "dimension": "What", "disambiguated_name": "$metadata.info.device", "type": "Sensor" }, { "dimension": "What", "disambiguated_name": "$metadata.info.alert", "type": "AlertType" } ], "publishedDate": "$SCRIPT( return _doc.metadata.info[0].date; )", "script": "", "scriptEngine": "javascript", "title": "$metadata.info.alert @ $metadata.info.date [$metadata.info.device]: $metadata.info.dstIP -> $metadata.info.srcIP" }, "tags": [ "cyber", "structured" ], "title": "Cyber Logs Test", "unstructuredAnalysis": { "meta": [ { "context": "First", "fieldName": "info", "script": "var info = decode(text); info;", "scriptlang": "javascript" } ], "script": "function decode(x)\n{\n var info = {}; \n var rec = x.split(','); \n info.device = rec[0];\n info.date = rec[1];\n info.srcIP = rec[2];\n info.dstIP = rec[3];\n info.alert = rec[4];\n info.country = rec[5];\n return info;\n}", "simpleTextCleanser": [ { "field": "fullText", "flags": "md", "replacement": " , ", "script": ",", "scriptlang": "regex" }, { "field": "description", "flags": "md", "replacement": " , ", "script": ",", "scriptlang": "regex" } ] }, "useExtractor":"none", "useTextExtractor":"none" } |
...