Input format sample
| Code Block | ||
|---|---|---|
| ||
Date,Device,SrcIP,dstIP,Alert,Country SCANNER_1,2012-01-01T13:43:00,10.0.0.1,66.66.66.66,DUMMY_ALERT_TYPE_1,United States SCANNER_2,2012-02-01T14:21:00,SCANNER_2,10.0.0.2,66.66.66.66,DUMMY_ALERT_TYPE_2,United Kingdom SCANNER_3,2012-03-01T15:17:00,10.0.0.1,99.66.99.66,DUMMY_ALERT_TYPE_3,Netherlands |
Source #1 - fileshare
| Code Block | ||
|---|---|---|
| ||
{
"description": "For cyber demo",
"extractType": "Feed",
"file": {
"XmlRootLevelValues": [],
"domain": "DOMAIN",
"password": "PASSWORD",
"type": "csv",
"username": "USER"
},
"isPublic": false,
"mediaType": "Log",
"searchCycle_secs": 3600,
"searchIndexFilter": {
"metadataFieldList": ""
},
"structuredAnalysis": {
"associations": [
{
"entity1": "$metadata.info.dstIP",
"entity2": "$metadata.info.srcIP",
"geo_index": "$SCRIPT( return _doc.metadata.info[0].country + '/country'; )",
"time_start": "$SCRIPT( return _doc.metadata.info[0].date; )",
"verb": "$SCRIPT( return _doc.metadata.info[0].alert; )",
"verb_category": "$SCRIPT( return _doc.metadata.info[0].alert; )"
}
],
"entities": [
{
"dimension": "What",
"disambiguated_name": "$metadata.info.srcIP",
"type": "PrivateIP"
},
{
"dimension": "What",
"disambiguated_name": "$metadata.info.dstIP",
"geotag": {
"country": "$SCRIPT( return _doc.metadata.info[0].country; )"
},
"ontology_type": "country",
"type": "PublicIP"
},
{
"actual_name": "$metadata.info.country",
"dimension": "Where",
"disambiguated_name": "$SCRIPT( return _doc.metadata.info[0].country; )",
"geotag": {
"country": "$SCRIPT( return _doc.metadata.info[0].country; )"
},
"ontology_type": "country",
"type": "Country"
},
{
"dimension": "What",
"disambiguated_name": "$metadata.info.device",
"type": "Sensor"
},
{
"dimension": "What",
"disambiguated_name": "$metadata.info.alert",
"type": "AlertType"
}
],
"publishedDate": "$SCRIPT( return _doc.metadata.info[0].date; )",
"script": "",
"scriptEngine": "javascript",
"title": "$metadata.info.alert @ $metadata.info.date [$metadata.info.device]: $metadata.info.dstIP -> $metadata.info.srcIP"
},
"tags": [
"cyber",
"structured"
],
"title": "Cyber Logs Test",
"unstructuredAnalysis": {
"meta": [
{
"context": "First",
"fieldName": "info",
"script": "var info = decode(text); info;",
"scriptlang": "javascript"
}
],
"script": "function decode(x)\n{\n var info = {}; \n var rec = x.split(','); \n info.device = rec[0];\n info.date = rec[1];\n info.srcIP = rec[2];\n info.dstIP = rec[3];\n info.alert = rec[4];\n info.country = rec[5];\n return info;\n}",
"simpleTextCleanser": [
{
"field": "fullText",
"flags": "md",
"replacement": " , ",
"script": ",",
"scriptlang": "regex"
},
{
"field": "description",
"flags": "md",
"replacement": " , ",
"script": ",",
"scriptlang": "regex"
}
]
},
"useExtractor":"none",
"useTextExtractor":"none",
"url": "smb://FILESHARE:139/cyber_logs/"
} |
...
It is slightly more complicated to parse CSV files over the Web, but still quite possible, using the searchConfig capability. Note that one neat trick is to upload a share to Infinit.e, and then use an API key to access the REST interface. Users can allocate themselves an API key from the People Manager.
| Info |
|---|
Note that when accessing Web documents you must use "rss.extraUrls" and specify minimally "url" and "title" fields, and not the top-level "url" (otherwise the URL is treated as an RSS feed rather than a standalone web page) |
| Code Block | ||
|---|---|---|
| ||
{
"description": "For cyber demo",
"extractType": "File",
"isPublic": false,
"mediaType": "Log",
"searchCycle_secs": 3600,
"rss": {
"extraUrls": [
{ "url": "http://INFINITE_ENDPOINT/api/share/get/51ad28a440b4a4f0f757824c?infinite_api_key=API_KEY" }
],
"searchConfig": {"script": "var retVals = [];\nvar n = -1;\nvar url = _doc.url.replace(/[?].*/,\"\");\nvar start = 0;\nwhile (start < text.length) {\n var end = text.indexOf('\\n', start);\n if (end == -1) end = text.length;\n var line = text.substr(start,end-1);\n start = end + 1; \n \n n++;\n if (0 == n) continue;\n \n var title = 'line #' + n.toString();\n var url2 = url + '#' + n.toString();\n var fullText = line;\n var retVal = { 'title':title, 'url':url2, 'fullText':line };\n retVals.push(retVal); \n}\nretVals;\n "}
},
"searchIndexFilter": {
"metadataFieldList": ""
},
"structuredAnalysis": {
"associations": [
{
"entity1": "$metadata.info.dstIP",
"entity2": "$metadata.info.srcIP",
"geo_index": "$SCRIPT( return _doc.metadata.info[0].country + '/country'; )",
"time_start": "$SCRIPT( return _doc.metadata.info[0].date; )",
"verb": "$SCRIPT( return _doc.metadata.info[0].alert; )",
"verb_category": "$SCRIPT( return _doc.metadata.info[0].alert; )"
}
],
"entities": [
{
"dimension": "What",
"disambiguated_name": "$metadata.info.srcIP",
"type": "PrivateIP"
},
{
"dimension": "What",
"disambiguated_name": "$metadata.info.dstIP",
"geotag": {
"country": "$SCRIPT( return _doc.metadata.info[0].country; )"
},
"ontology_type": "country",
"type": "PublicIP"
},
{
"actual_name": "$metadata.info.country",
"dimension": "Where",
"disambiguated_name": "$SCRIPT( return _doc.metadata.info[0].country; )",
"geotag": {
"country": "$SCRIPT( return _doc.metadata.info[0].country; )"
},
"ontology_type": "country",
"type": "Country"
},
{
"dimension": "What",
"disambiguated_name": "$metadata.info.device",
"type": "Sensor"
},
{
"dimension": "What",
"disambiguated_name": "$metadata.info.alert",
"type": "AlertType"
}
],
"publishedDate": "$SCRIPT( return _doc.metadata.info[0].date; )",
"script": "",
"scriptEngine": "javascript",
"title": "$metadata.info.alert @ $metadata.info.date [$metadata.info.device]: $metadata.info.dstIP -> $metadata.info.srcIP"
},
"tags": [
"cyber",
"structured"
],
"title": "Cyber Logs Test",
"unstructuredAnalysis": {
"meta": [
{
"context": "First",
"fieldName": "info",
"script": "var info = decode(text); info;",
"scriptlang": "javascript"
}
],
"script": "function decode(x)\n{\n var info = {}; \n var rec = x.split(','); \n info.device = rec[0];\n info.date = rec[1];\n info.srcIP = rec[2];\n info.dstIP = rec[3];\n info.alert = rec[4];\n info.country = rec[5];\n return info;\n}",
"simpleTextCleanser": [
{
"field": "fullText",
"flags": "md",
"replacement": " , ",
"script": ",",
"scriptlang": "regex"
},
{
"field": "description",
"flags": "md",
"replacement": " , ",
"script": ",",
"scriptlang": "regex"
}
]
},
"useExtractor":"none",
"useTextExtractor":"none"
} |
...