Versions Compared

Old Version 6

changes.mady.by.user AlexI

Saved on

compared with

New Version Current

changes.mady.by.user AlexI

Saved on

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.

Input format sample

Code Block
languagejavascript
Date,Device,SrcIP,dstIP,Alert,Country
SCANNER_1,2012-01-01T13:43:00,10.0.0.1,66.66.66.66,DUMMY_ALERT_TYPE_1,United States
SCANNER_2,2012-02-01T14:21:00,SCANNER_2,10.0.0.2,66.66.66.66,DUMMY_ALERT_TYPE_2,United Kingdom
SCANNER_3,2012-03-01T15:17:00,10.0.0.1,99.66.99.66,DUMMY_ALERT_TYPE_3,Netherlands

Source #1 #1a - fileshare, manual parsing

Code Block
languagejavascript
{
    "description": "For cyber demo",
    "extractTypeisPublic": "Feed"false,
    "filemediaType": {"Log",
    "searchCycle_secs": 3600,
    "XmlRootLevelValuestags": [],
        "domain": "DOMAIN"cyber",
        "passwordstructured":
"PASSWORD",    ],
    "typetitle": "csvCyber Logs Test",
 
      "usernameprocessingPipeline": "USER"[
    },    {
"isPublic": false,     "mediaType": "Log",     "searchCycle_secsfile": {
3600,      "searchIndexFilter": {         "metadataFieldListXmlRootLevelValues": ""[],
      },     "structuredAnalysis": {    "domain": "DOMAIN",
   "associations": [             {"password": "PASSWORD",
                "entity1type": "$metadata.info.dstIPcsv",
                "entity2username": "$metadata.info.srcIPUSER",
                "geo_indexurl": "$SCRIPT( return _doc.metadata.info[0].country + '/country'; )",smb://FILESHARE:139/cyber_logs/"
            }
    "time_start": "$SCRIPT( return _doc.metadata.info[0].date; )", },
        {
        "verb": "$SCRIPT( return _doc.metadata.info[0].alert; )", "globals": {
                "verb_categoryscripts": "$SCRIPT( return _doc.metadata.info[0].alert;
)"             }       "function  ],decode(x)\n{\n    var info = {};   \n "entities": [  var rec = x.split(',');   \n    info.device {
= rec[0];\n    info.date = rec[1];\n    info.srcIP = rec[2];\n    "dimension": "What",
info.dstIP = rec[3];\n    info.alert = rec[4];\n    info.country = rec[5];\n    "disambiguated_name": "$metadata.info.srcIP",return info;\n}"
                "type": "PrivateIP"]
            }
},        },
    {    {
            "dimensionharvest": "What",{
                "disambiguatedsearchCycle_namesecs": "$metadata.info.dstIP", 3600
            }
        "geotag": {},
        {
            "countrydocMetadata": "$SCRIPT( return _doc.metadata.info[0].country; )" {
                "title": "$metadata.info.alert @ $metadata.info.date [$metadata.info.device]: $metadata.info.dstIP  }-> $metadata.info.srcIP",
                "ontology_typepublishedDate": "country",$SCRIPT( return _doc.metadata.info[0].date; )"
            }
      "type": "PublicIP"  },
        {
   },         "contentMetadata": [
  {              {
  "actual_name": "$metadata.info.country",                 "dimensionfieldName": "Whereinfo",
                    "disambiguated_namescript": "$SCRIPT( return _doc.metadata.info[0].country; )",var info = decode(text); info;",
                    "geotagscriptlang": {"javascript"
                }
      "country": "$SCRIPT( return _doc.metadata.info[0].country; )"  ]
         },
    },    {
            "ontology_typetext": "country",[
                "type": "Country"{
             },         "fieldName": "fullText",
   {                 "dimensionscript": "What,",
                 "disambiguated_name   "scriptlang": "$metadata.info.deviceregex",
                    "typeflags": "Sensormd",
            },        "replacement": " , "
 {               },
 "dimension": "What",              {
  "disambiguated_name": "$metadata.info.alert",                 "typefieldName": "AlertTypedescription",
            }         ]"script": ",",
        "publishedDate": "$SCRIPT( return _doc.metadata.info[0].date; )",         "scriptscriptlang": "regex",
          "scriptEngine": "javascript",         "titleflags": "$metadata.info.alert @ $metadata.info.date [$metadata.info.device]: $metadata.info.dstIP -> $metadata.info.srcIP""md",
               },     "tagsreplacement": [" , "
      "cyber",         "structured" }
   ],     "title": "Cyber Logs Test", ]
   "unstructuredAnalysis": {    },
    "meta    {
            "entities": [
                {
                    "contextdimension": "FirstWhat",
                    "fieldNamedisambiguated_name": "$metadata.info.srcIP",
                    "scripttype": "PrivateIP"var
  info = decode(text); info;",           },
     "scriptlang": "javascript"          {
  }         ],         "scriptdimension": "What"function decode(x)\n{\n,
    var info = {};   \n    var rec = x.split(',');   \n    info.device = rec[0];\n"disambiguated_name": "$metadata.info.dstIP",
      info.date = rec[1];\n    info.srcIP = rec[2];\n    info.dstIP = rec[3];\n  "geotag": {
  info.alert = rec[4];\n    info.country = rec[5];\n    return info;\n}",         "simpleTextCleansercountry": "$SCRIPT( return _doc.metadata.info[0].country; )"
            {        },
        "field": "fullText",
                "flags"ontology_type": "mdcountry",
                    "replacementtype": " , PublicIP",
                "script": ","},
                "scriptlang": "regex"{
             },       "actual_name": "$metadata.info.country",
    {
                "fielddimension": "descriptionWhere",
                 "flags   "disambiguated_name": "md",$SCRIPT( return _doc.metadata.info[0].country; )",
             "replacement": " , ",    "geotag": {
           "script": ",",            "country": "$SCRIPT( return _doc.metadata.info[0].country; )"
    "scriptlang": "regex"                },
          ]     },     "useExtractorontology_type": "nonecountry",
           "useTextExtractor         "type": "none",Country"
              "url": "smb://FILESHARE:139/cyber_logs/" }

Source #2 - web (including uploaded fileshares)

It is slightly more complicated to parse CSV files over the Web, but still quite possible, using the searchConfig capability. Note that one neat trick is to upload a share to Infinit.e, and then use an API key to access the REST interface. Users can allocate themselves an API key from the People Manager.

Info

Note that when accessing Web documents you must use "rss.extraUrls" and specify minimally "url" and "title" fields, and not the top-level "url" (otherwise the URL is treated as an RSS feed rather than a standalone web page)

Code Block
languagejavascript
{,
     "description": "For cyber demo",         {
        "extractType": "File",     "isPublic": false,     "mediaTypedimension": "LogWhat",
       "searchCycle_secs": 3600,        "rss": { 		"extraUrls": [ 			{ "urldisambiguated_name": "http://INFINITE_ENDPOINT/api/share/get/51ad28a440b4a4f0f757824c?infinite_api_key=API_KEY" }
		],$metadata.info.device",
                    "searchConfigtype": {"scriptSensor":
"var retVals = [];\nvar n = -1;\nvar url = _doc.url.replace(/[?].*/,\"\");\nvar start = 0;\nwhile (start < text.length) {\n },
   var end = text.indexOf('\\n', start);\n    if (end == -1) end = text.length;\n{
    var line = text.substr(start,end-1);\n    start = end + 1;    \n  "dimension": "What",
  \n       n++;\n    if (0 == n) continue;\n    \n"disambiguated_name": "$metadata.info.alert",
   var title = 'line #' + n.toString();\n    var url2 = url + '#' + n.toString();\n  "type": "AlertType"
  var fullText = line;\n    var retVal = { 'title':title, 'url':url2, 'fullText':line };\n
    retVals.push(retVal); \n}\nretVals;\n        "}]
    },    },
"searchIndexFilter": {       {
 "metadataFieldList": ""     },     "structuredAnalysisassociations": {
[
       "associations": [         {
   {                 "entity1": "$metadata.info.dstIP",
                    "entity2": "$metadata.info.srcIP",
                    "geo_index": "$SCRIPT( return _doc.metadata.info[0].country + '/country'; )",
                    "time_start": "$SCRIPT( return _doc.metadata.info[0].date; )",
                    "verb": "$SCRIPT( return _doc.metadata.info[0].alert; )",
                    "verb_category": "$SCRIPT( return _doc.metadata.info[0].alert; )"
            }    }
    ],         "entities":]
[        },
    {    {
            "dimensionsearchIndex": "What",{
                "disambiguated_namemetadataFieldList": "$metadata.info.srcIP",
            }
   "type": "PrivateIP"    }
       ]
}

Source #1b - fileshare, automated parsing - headers manually specified

Code Block
languagejavascript
{
    "description": "For cyber demo",
     {"isPublic": false,
    "mediaType": "Log",
    "searchCycle_secs": 3600,
    "dimensiontags": [
"What",        "cyber",
        "disambiguated_namestructured":
"$metadata.info.dstIP",    ],
    "title": "Cyber Logs Test",
    "geotagprocessingPipeline": {[
        {
            "countryfile": "$SCRIPT({
return _doc.metadata.info[0].country; )"              "XmlRootLevelValues": [
 },                 "ontology_type":  "countrydevice",
                  "type":  "PublicIPdate",
            },         "srcIP",
   {                 "actual_name": "$metadata.info.country",dstIP",
                   "dimension": "Wherealert",
                 "disambiguated_name": "$SCRIPT( return _doc.metadata.info[0].country; )", "country"
                "geotag": {],
                    "countryXmlIgnoreValues": "$SCRIPT([
 return _doc.metadata.info[0].country; )"                 },"device,date,srcIP"
                "ontology_type": "country",
],
                "typedomain": "CountryDOMAIN",
            },             {"password": "PASSWORD",
                "dimensiontype": "Whatcsv",
                "disambiguated_nameusername": "$metadata.info.deviceUSER",
                "typeurl": "Sensorsmb://FILESHARE:139/cyber_logs/"
            },
        },
   {     {
            "dimensionharvest": "What",{
                "disambiguatedsearchCycle_namesecs": "$metadata.info.alert", 3600
            }
     "type": "AlertType"  },
        {
 }         ],  "docMetadata": {
     "publishedDate": "$SCRIPT( return _doc.metadata.info[0].date; )",
        "script": "",
        "scriptEngine": "javascript",
        "title": "$metadata.infocsv.alert @ $metadata.infocsv.date [$metadata.infocsv.device]: $metadata.infocsv.dstIP -> $metadata.infocsv.srcIP",
         },       "tagspublishedDate": "$SCRIPT( return _doc.metadata.csv[0].date;         "cyber",
        "structured"
    ],
    "title": "Cyber Logs Test",
    "unstructuredAnalysis": {
        "meta": [
            {
                "context": "First",
                "fieldName": "info",
                "script": "var info = decode(text); info;",
                "scriptlang": "javascript"
            }
        ],
        "script": "function decode(x)\n{\n    var info = {};   \n    var rec = x.split(',');   \n    info.device = rec[0];\n    info.date = rec[1];\n    info.srcIP = rec[2];\n    info.dstIP = rec[3];\n    info.alert = rec[4];\n    info.country = rec[5];\n    return info;\n}",
        "simpleTextCleanser": [
            {
                "field": "fullText",)"
            }
        },
        {
            "entities": [
                {
                    "dimension": "What",
                    "disambiguated_name": "$metadata.csv.srcIP",
                    "type": "PrivateIP"
                },
                {
                    "dimension": "What",
                    "disambiguated_name": "$metadata.csv.dstIP",
                    "geotag": {
                        "country": "$SCRIPT( return _doc.metadata.csv[0].country; )"
                    },
                    "ontology_type": "country",
                    "type": "PublicIP"
                },
                {
                    "actual_name": "$metadata.csv.country",
                    "dimension": "Where",
                    "disambiguated_name": "$SCRIPT( return _doc.metadata.csv[0].country; )",
                    "geotag": {
                        "country": "$SCRIPT( return _doc.metadata.csv[0].country; )"
                    },
                    "ontology_type": "country",
                    "type": "Country"
                },
                {
                    "dimension": "What",
                    "disambiguated_name": "$metadata.csv.device",
                    "type": "Sensor"
                },
                {
                    "dimension": "What",
                    "disambiguated_name": "$metadata.csv.alert",
                    "type": "AlertType"
                }
            ]
        },
        {
            "associations": [
                {
                    "entity1": "$metadata.csv.dstIP",
                    "entity2": "$metadata.csv.srcIP",
                    "geo_index": "$SCRIPT( return _doc.metadata.csv[0].country + '/country'; )",
                    "time_start": "$SCRIPT( return _doc.metadata.csv[0].date; )",
                    "verb": "$SCRIPT( return _doc.metadata.csv[0].alert; )",
                    "verb_category": "$SCRIPT( return _doc.metadata.csv[0].alert; )"
                }
            ]
        },
        {
            "searchIndex": {
                "metadataFieldList": ""
            }
        }
    ]
}

Source #1c - fileshare, automated parsing - headers automatically specified

For the purpose of this example, imagine that the first line starts with a "#", eg:

Code Block
languagejavascript
#Date,Device,SrcIP,dstIP,Alert,Country
SCANNER_1,2012-01-01T13:43:00,10.0.0.1,66.66.66.66,DUMMY_ALERT_TYPE_1,United States

...

Then:

Code Block
languagejavascript
{
    "description": "For cyber demo",
    "isPublic": false,
    "mediaType": "Log",
    "searchCycle_secs": 3600,
    "tags": [
        "cyber",
        "structured"
    ],
    "title": "Cyber Logs Test",
    "processingPipeline": [
        {
            "file": {
                "XmlIgnoreValues": [
                    "#"
                ],
                "domain": "DOMAIN",
                "password": "PASSWORD",
                "type": "csv",
                "username": "USER",
                "url": "smb://FILESHARE:139/cyber_logs/"
            }
        },
        {
            "harvest": {
                "searchCycle_secs": 3600
            }
        },
        {
            "docMetadata": {
                "title": "$metadata.csv.alert @ $metadata.csv.date [$metadata.csv.device]: $metadata.csv.dstIP -> $metadata.csv.srcIP",
                "publishedDate": "$SCRIPT( return _doc.metadata.csv[0].date; )"
            }
        },
        {
            "entities": [
                {
                    "dimension": "What",
                    "disambiguated_name": "$metadata.csv.srcIP",
                    "type": "PrivateIP"
                },
                {
                    "dimension": "What",
                    "disambiguated_name": "$metadata.csv.dstIP",
                    "geotag": {
                        "country": "$SCRIPT( return _doc.metadata.csv[0].country; )"
                    },
                    "ontology_type": "country",
                    "type": "PublicIP"
                },
                {
                    "actual_name": "$metadata.csv.country",
                    "dimension": "Where",
                    "disambiguated_name": "$SCRIPT( return _doc.metadata.csv[0].country; )",
                    "geotag": {
                        "country": "$SCRIPT( return _doc.metadata.csv[0].country; )"
                    },
                    "ontology_type": "country",
                    "type": "Country"
                },
                {
                    "dimension": "What",
                    "disambiguated_name": "$metadata.csv.device",
                    "type": "Sensor"
                },
                {
                    "dimension": "What",
                    "disambiguated_name": "$metadata.csv.alert",
                    "type": "AlertType"
                }
            ]
        },
        {
            "associations": [
                {
                    "entity1": "$metadata.csv.dstIP",
                    "entity2": "$metadata.csv.srcIP",
                    "geo_index": "$SCRIPT( return _doc.metadata.csv[0].country + '/country'; )",
                    "time_start": "$SCRIPT( return _doc.metadata.csv[0].date; )",
                    "verb": "$SCRIPT( return _doc.metadata.csv[0].alert; )",
                    "verb_category": "$SCRIPT( return _doc.metadata.csv[0].alert; )"
                }
            ]
        },
        {
            "searchIndex": {
                "metadataFieldList": ""
            }
        }
    ]
}

Note that if the header was as originally specified (eg first line "Date,Device,SrcIP,dstIP,Alert,Country"), then the "XmlIgnoreValues" field would have need to be "\"Device\"" to correctly parse the headers (assuming the quote field was ").

Source #2a - web (including uploaded fileshares), manual parsing

It is slightly more complicated to parse CSV files over the Web, but still quite possible, using the searchConfig capability. Note that one neat trick is to upload a share to Infinit.e, and then use an API key to access the REST interface. Users can allocate themselves an API key from the People Manager.

Info

Note that when accessing Web documents you must use "rss.extraUrls" and specify minimally "url" and "title" fields, and not the top-level "url" (otherwise the URL is treated as an RSS feed rather than a standalone web page)

Code Block
languagejavascript
{
    "description": "For cyber demo",
    "isPublic": false,
    "mediaType": "Log",
    "searchCycle_secs": 3600,
    "tags": [
        "cyber",
        "structured"
    ],
    "title": "Cyber Logs Test",
    "processingPipeline": [
        {
            "feed": {
                "extraUrls": [
                    {
                        "url": "http://INFINITE_ENDPOINT/api/share/get/51ad28a440b4a4f0f757824c?infinite_api_key=API_KEY"
                    }
                ]
            }
        },
        {
            "globals": {
                "scripts": [
                    "function decode(x)\n{\n    var info = {};   \n    var rec = x.split(',');   \n    info.device = rec[0];\n    info.date = rec[1];\n    info.srcIP = rec[2];\n    info.dstIP = rec[3];\n    info.alert = rec[4];\n    info.country = rec[5];\n    return info;\n}"
                ]
            }
        },
        {
            "harvest": {
                "searchCycle_secs": 3600
            }
        },
        {
            "links": {
                "script": "var retVals = [];\nvar n = -1;\nvar url = _doc.url.replace(/[?].*/,\"\");\nvar start = 0;\nwhile (start < text.length) {\n    var end = text.indexOf('\\n', start);\n    if (end == -1) end = text.length;\n    var line = text.substr(start,end-1);\n    start = end + 1;    \n    \n    n++;\n    if (0 == n) continue;\n    \n    var title = 'line #' + n.toString();\n    var url2 = url + '#' + n.toString();\n    var fullText = line;\n    var retVal = { 'title':title, 'url':url2, 'fullText':line };\n    retVals.push(retVal); \n}\nretVals;\n      "
            }
        },
        {
            "docMetadata": {
                "title": "$metadata.info.alert @ $metadata.info.date [$metadata.info.device]: $metadata.info.dstIP -> $metadata.info.srcIP",
                "publishedDate": "$SCRIPT( return _doc.metadata.info[0].date; )"
            }
        },
        {
            "contentMetadata": [
                {
                    "fieldName": "info",
                    "script": "var info = decode(text); info;",
                    "scriptlang": "javascript"
                }
            ]
        },
        {
            "text": [
                {
                    "fieldName": "fullText",
                    "script": ",",
                    "scriptlang": "regex",
                    "flags": "md",
                    "replacement": " , "
                },
                {
                    "fieldName": "description",
                    "script": ",",
                    "scriptlang": "regex",
                    "flags": "md",
                    "replacement": " , "
                }
            ]
        },
        {
            "entities": [
                {
                    "dimension": "What",
                    "disambiguated_name": "$metadata.info.srcIP",
                    "type": "PrivateIP"
                },
                {
                    "dimension": "What",
                    "disambiguated_name": "$metadata.info.dstIP",
                    "geotag": {
                        "country": "$SCRIPT( return _doc.metadata.info[0].country; )"
                    },
                    "ontology_type": "country",
                    "type": "PublicIP"
                },
                {
                    "actual_name": "$metadata.info.country",
                    "dimension": "Where",
                    "disambiguated_name": "$SCRIPT( return _doc.metadata.info[0].country; )",
                    "geotag": {
                        "country": "$SCRIPT( return _doc.metadata.info[0].country; )"
                    },
                    "ontology_type": "country",
                    "type": "Country"
                },
                {
                    "dimension": "What",
                    "disambiguated_name": "$metadata.info.device",
                    "type": "Sensor"
                },
                {
                    "dimension": "What",
                    "disambiguated_name": "$metadata.info.alert",
                    "type": "AlertType"
                }
            ]
        },
        {
            "associations": [
                {
                    "entity1": "$metadata.info.dstIP",
                    "entity2": "$metadata.info.srcIP",
                    "geo_index": "$SCRIPT( return _doc.metadata.info[0].country + '/country'; )",
                    "time_start": "$SCRIPT( return _doc.metadata.info[0].date; )",
                    "verb": "$SCRIPT( return _doc.metadata.info[0].alert; )",
                    "verb_category": "$SCRIPT( return _doc.metadata.info[0].alert; )"
                }
            ]
        },
        {
            "searchIndex": {
                "metadataFieldList": ""
            }
        }
    ]
}

Note that the API key is not visible in any of the extracted documents (it is removed in the "searchConfig.script" code), and is also not visible in the source to anyone but the source owner and administrator (due to the "isPublic:false" field). As an alternative (from June 2013), a cookie can be used: (eg) "rss.httpFields": { "Cookie": "infinitecookie=api:API_KEY;" }.

Source #2b - web (including uploaded fileshares), manual parsing using Java

As above, except "unstructuredAnalysis.script" will look like:

Code Block
var parser = new Packages.au.com.bytecode.opencsv.CSVParser();
function decode(x)
{
   var rec = parser.parseLine(x.toString());
   var info = {};   
   info.device = '' + rec[0];
   info.date = '' + rec[1];
   info.srcIP = '' + rec[2];
   info.dstIP = '' + rec[3];
   info.alert = '' + rec[4];
   info.country = '' + rec[5];
   return info;
}

(Note that the "'' +  <string-variable>" construct is necessary to convert from Java strings to javascript strings)

See the opencsv documentation for more details.

Output sample

(For source 1b, metadata.info is called metadata.csv)
Code Block
languagejavascript
{
    "associations": [{
        "assoc_type": "Event",
        "entity1": "66.66.66.66",
        "entity1_index": "66.66.66.66/publicip",
        "entity2": "10.0.0.1",
        "entity2_index": "10.0.0.1/privateip",
        "geo_index": "united states/country",
        "time_start": "2012-01-01T13:43:00",
        "verb": "DUMMY_ALERT_TYPE_1",
        "verb_category": "DUMMY_ALERT_TYPE_1"
    }],
    "communityId": ["506dc16dfbf042893dd6b8f2"],
    "created": "Jun 4, 2013 12:54:34 AM UTC",
    "entities": [
        {
            "actual_name": "10.0.0.1",
            "dimension": "What",
            "disambiguated_name": "10.0.0.1",
            "doccount": 0,
            "frequency": 1,
            "index": "10.0.0.1/privateip",
            "relevance": 0,
            "totalfrequency": -1,
            "type": "PrivateIP"
        },
        {
            "actual_name": "66.66.66.66",
            "dimension": "What",
            "disambiguated_name": "66.66.66.66",
            "doccount": 0,
            "frequency": 1,
            "index": "66.66.66.66/publicip",
            "relevance": 0,
            "totalfrequency": -1,
            "type": "PublicIP"
        },
        {
            "actual_name": "United States",
            "dimension": "Where",
            "disambiguated_name": "United States",
            "doccount": 0,
            "frequency": 1,
            "index": "united states/country",
            "ontology_type": "country",
            "relevance": 0,
            "totalfrequency": -1,
            "type": "Country"
        },
        {
            "actual_name": "SCANNER_1",
            "dimension": "What",
            "disambiguated_name": "SCANNER_1",
            "doccount": 0,
            "frequency": 1,
            "index": "scanner_1/sensor",
            "relevance": 0,
            "totalfrequency": -1,
            "type": "Sensor"
        },
        {
            "actual_name": "DUMMY_ALERT_TYPE_1",
            "dimension": "What",
            "disambiguated_name": "DUMMY_ALERT_TYPE_1",
            "doccount": 0,
            "frequency": 1,
            "index": "dummy_alert_type_1/alerttype",
            "relevance": 0,
            "totalfrequency": -1,
            "type": "AlertType"
        }
    ],
    "fullText": "SCANNER_1 , 2012-01-01T13:43:00 , 10.0.0.1 , 66.66.66.66 , DUMMY_ALERT_TYPE_1 , United States",
    "mediaType": ["Log"],
    "metadata": {"info": [{
        "alert": "DUMMY_ALERT_TYPE_1 ",
        "country": "United States",
        "date": "2012-01-01T13:43:00",
        "device": "SCANNER_1 ",
        "dstIP": "66.66.66.66",
        "srcIP": " 10.0.0.1"
    }]},
    "modified": "Jun 4, 2013 12:54:34 AM UTC",
    "publishedDate": "January 1, 2012 13:43:00 PM UTC",
    "source": ["Cyber Logs Test"],
    "sourceKey": ["INFINITE_ENDPOINT.api.share.get.51ad28a440b4a4f0f757824c.25.26"],
    "tags": [
        "cyber",
        "structured"
    ],
    "title": "DUMMY_ALERT_TYPE_1  @ 2012-01-01T13:43:00 [SCANNER_1 ]: 66.66.66.66 -> 10.0.0.1",
    "url": "http://INFINITE_ENDPOINT/api/share/get/51ad28a440b4a4f0f757824c#1"
}

Sources - old format

Old Format Source #1a - fileshare, manual parsing

Code Block
languagejavascript
{
    "description": "For cyber demo",
    "extractType": "File",
    "file": {
        "XmlRootLevelValues": [],
        "domain": "DOMAIN",
        "password": "PASSWORD",
        "type": "csv",
        "username": "USER"
    },
    "isPublic": false,
    "mediaType": "Log",
    "searchCycle_secs": 3600,
    "searchIndexFilter": {
        "metadataFieldList": ""
    },
    "structuredAnalysis": {
        "associations": [
            {
                "entity1": "$metadata.info.dstIP",
                "entity2": "$metadata.info.srcIP",
                "geo_index": "$SCRIPT( return _doc.metadata.info[0].country + '/country'; )",
                "time_start": "$SCRIPT( return _doc.metadata.info[0].date; )",
                "verb": "$SCRIPT( return _doc.metadata.info[0].alert; )",
                "verb_category": "$SCRIPT( return _doc.metadata.info[0].alert; )"
            }
        ],
        "entities": [
            {
                "dimension": "What",
                "disambiguated_name": "$metadata.info.srcIP",
                "type": "PrivateIP"
            },
            {
                "dimension": "What",
                "disambiguated_name": "$metadata.info.dstIP",
                "geotag": {
                    "country": "$SCRIPT( return _doc.metadata.info[0].country; )"
                },
                "ontology_type": "country",
                "type": "PublicIP"
            },
            {
                "actual_name": "$metadata.info.country",
                "dimension": "Where",
                "disambiguated_name": "$SCRIPT( return _doc.metadata.info[0].country; )",
                "geotag": {
                    "country": "$SCRIPT( return _doc.metadata.info[0].country; )"
                },
                "ontology_type": "country",
                "type": "Country"
            },
            {
                "dimension": "What",
                "disambiguated_name": "$metadata.info.device",
                "type": "Sensor"
            },
            {
                "dimension": "What",
                "disambiguated_name": "$metadata.info.alert",
                "type": "AlertType"
            }
        ],
        "publishedDate": "$SCRIPT( return _doc.metadata.info[0].date; )",
        "script": "",
        "scriptEngine": "javascript",
        "title": "$metadata.info.alert @ $metadata.info.date [$metadata.info.device]: $metadata.info.dstIP -> $metadata.info.srcIP"
    },
    "tags": [
        "cyber",
        "structured"
    ],
    "title": "Cyber Logs Test",
    "unstructuredAnalysis": {
        "meta": [
            {
                "context": "First",
                "fieldName": "info",
                "script": "var info = decode(text); info;",
                "scriptlang": "javascript"
            }
        ],
        "script": "function decode(x)\n{\n    var info = {};   \n    var rec = x.split(',');   \n    info.device = rec[0];\n    info.date = rec[1];\n    info.srcIP = rec[2];\n    info.dstIP = rec[3];\n    info.alert = rec[4];\n    info.country = rec[5];\n    return info;\n}",
        "simpleTextCleanser": [
            {
                "field": "fullText",
                "flags": "md",
                "replacement": " , ",
                "script": ",",
                "scriptlang": "regex"
            },
            {
                "field": "description",
                "flags": "md",
                "replacement": " , ",
                "script": ",",
                "scriptlang": "regex"
            }
        ]
    },
    "useExtractor":"none",
    "useTextExtractor":"none",
    "url": "smb://FILESHARE:139/cyber_logs/"
}

Old Format Source #1b - fileshare, automated parsing

Code Block
languagejavascript
{
    "description": "For cyber demo",
    "extractType": "File",
    "file": {
        "XmlRootLevelValues": [ "device", "date", "srcIP", "dstIP", "alert", "country" ],
		"XmlIgnoreValues": [ "device,date,srcIP" ],
        "domain": "DOMAIN",
        "password": "PASSWORD",
        "type": "csv",
        "username": "USER"
    },
    "isPublic": false,
    "mediaType": "Log",
    "searchCycle_secs": 3600,
    "searchIndexFilter": {
        "metadataFieldList": ""
    },
    "structuredAnalysis": {
        "associations": [
            {
                "entity1": "$metadata.csv.dstIP",
                "entity2": "$metadata.csv.srcIP",
                "geo_index": "$SCRIPT( return _doc.metadata.csv[0].country + '/country'; )",
                "time_start": "$SCRIPT( return _doc.metadata.csv[0].date; )",
                "verb": "$SCRIPT( return _doc.metadata.csv[0].alert; )",
                "verb_category": "$SCRIPT( return _doc.metadata.csv[0].alert; )"
            }
        ],
        "entities": [
            {
                "dimension": "What",
                "disambiguated_name": "$metadata.csv.srcIP",
                "type": "PrivateIP"
            },
            {
                "dimension": "What",
                "disambiguated_name": "$metadata.csv.dstIP",
                "geotag": {
                    "country": "$SCRIPT( return _doc.metadata.csv[0].country; )"
                },
                "ontology_type": "country",
                "type": "PublicIP"
            },
            {
                "actual_name": "$metadata.csv.country",
                "dimension": "Where",
                "disambiguated_name": "$SCRIPT( return _doc.metadata.csv[0].country; )",
                "geotag": {
                    "country": "$SCRIPT( return _doc.metadata.csv[0].country; )"
                },
                "ontology_type": "country",
                "type": "Country"
            },
            {
                "dimension": "What",
                "disambiguated_name": "$metadata.csv.device",
                "type": "Sensor"
            },
            {
                "dimension": "What",
                "disambiguated_name": "$metadata.csv.alert",
                "type": "AlertType"
            }
        ],
        "publishedDate": "$SCRIPT( return _doc.metadata.csv[0].date; )",
        "script": "",
        "scriptEngine": "javascript",
        "title": "$metadata.csv.alert @ $metadata.csv.date [$metadata.csv.device]: $metadata.csv.dstIP -> $metadata.csv.srcIP"
    },
    "tags": [
        "cyber",
        "structured"
    ],
    "title": "Cyber Logs Test",
    "useExtractor":"none",
    "useTextExtractor":"none",
    "url": "smb://FILESHARE:139/cyber_logs/"
}

Old Format Source #2a - web (including uploaded fileshares), manual parsing

It is slightly more complicated to parse CSV files over the Web, but still quite possible, using the searchConfig capability. Note that one neat trick is to upload a share to Infinit.e, and then use an API key to access the REST interface. Users can allocate themselves an API key from the People Manager.

Info

Note that when accessing Web documents you must use "rss.extraUrls" and specify minimally "url" and "title" fields, and not the top-level "url" (otherwise the URL is treated as an RSS feed rather than a standalone web page)

Code Block
languagejavascript
{
    "description": "For cyber demo",
    "extractType": "Feed",
    "isPublic": false,
    "mediaType": "Log",
    "searchCycle_secs": 3600,
    "rss": {
		"extraUrls": [
			{ "url": "http://INFINITE_ENDPOINT/api/share/get/51ad28a440b4a4f0f757824c?infinite_api_key=API_KEY" }
		],
        "searchConfig": {"script": "var retVals = [];\nvar n = -1;\nvar url = _doc.url.replace(/[?].*/,\"\");\nvar start = 0;\nwhile (start < text.length) {\n    var end = text.indexOf('\\n', start);\n    if (end == -1) end = text.length;\n    var line = text.substr(start,end-1);\n    start = end + 1;    \n    \n    n++;\n    if (0 == n) continue;\n    \n    var title = 'line #' + n.toString();\n    var url2 = url + '#' + n.toString();\n    var fullText = line;\n    var retVal = { 'title':title, 'url':url2, 'fullText':line };\n    retVals.push(retVal); \n}\nretVals;\n      "}
    },
    "searchIndexFilter": {
        "metadataFieldList": ""
    },
    "structuredAnalysis": {
        "associations": [
                "flags": "md",{
                "replacemententity1": " , $metadata.info.dstIP",
                "scriptentity2": ",$metadata.info.srcIP",
                "scriptlanggeo_index": "regex"$SCRIPT( return _doc.metadata.info[0].country           },
            {+ '/country'; )",
                "fieldtime_start": "description$SCRIPT( return _doc.metadata.info[0].date; )",
                "flagsverb": "md$SCRIPT( return _doc.metadata.info[0].alert; )",
                "replacementverb_category": " , ",$SCRIPT( return _doc.metadata.info[0].alert; )"
             }
  "script": ",",      ],
          "scriptlangentities": "regex"[
            }{
        ]     },
    "useExtractordimension": "none",What",
                "useTextExtractordisambiguated_name":"none"	 }

Note that the API key is not visible in any of the extracted documents (it is removed in the "searchConfig.script" code), and is also not visible in the source to anyone but the source owner and administrator (due to the "isPublic:false" field). As an alternative (from June 2013), a cookie can be used: (eg) "rss.httpFields": { "Cookie": "infinitecookie=api:API_KEY;" }.

Output sample

Code Block
languagejavascript
{"$metadata.info.srcIP",
      "associations": [{         "assoc_type": "Event",PrivateIP"
            },
            {
                "entity1dimension": "66.66.66.66What",
        "entity1_index        "disambiguated_name": "66$metadata.66.66.66/publicipinfo.dstIP",
         "entity2       "geotag": "10.0.0.1", {
         "entity2_index": "10.0.0.1/privateip",           "geo_indexcountry": "united states/country",$SCRIPT( return _doc.metadata.info[0].country; )"
             "time_start": "2012-01-01T13:43:00",  },
      "verb": "DUMMY_ALERT_TYPE_1",         "verbontology_categorytype": "DUMMY_ALERT_TYPE_1"country",
      }],          "communityIdtype": ["506dc16dfbf042893dd6b8f2PublicIP"],
      "created": "Jun 4, 2013 12:54:34 AM UTC"},
      "entities": [     {
   {             "actual_name": "10$metadata.0info.0.1country",
                "dimension": "WhatWhere",
                "disambiguated_name": "10.0.0.1",$SCRIPT( return _doc.metadata.info[0].country; )",
                "doccountgeotag": 0, {
                    "frequencycountry": 1 "$SCRIPT( return _doc.metadata.info[0].country; )"
                },
            "index    "ontology_type": "10.0.0.1/privateipcountry",
                "relevancetype": 0,"Country"
            },
 "totalfrequency": -1,          {
  "type": "PrivateIP"         },    "dimension": "What",
   {             "actualdisambiguated_name": "66$metadata.66info.66.66device",
                "dimensiontype": "What",Sensor"
            },
            {
  "disambiguated_name": "66.66.66.66",             "doccountdimension": 0"What",
             "frequency   "disambiguated_name": 1,"$metadata.info.alert",
                "indextype": "66.66.66.66/publicip",AlertType"
            }
   "relevance": 0,    ],
        "totalfrequencypublishedDate": -1,
   "$SCRIPT( return _doc.metadata.info[0].date; )",
        "typescript": "PublicIP"
        },
        {
   "scriptEngine": "javascript",
        "actual_nametitle": "United States",
 $metadata.info.alert @ $metadata.info.date [$metadata.info.device]: $metadata.info.dstIP -> $metadata.info.srcIP"
     },
    "dimensiontags": "Where",
   [
        "disambiguated_name": "United States"cyber",
        "structured"
   "doccount": 0],
    "title": "Cyber Logs Test",
    "frequencyunstructuredAnalysis": 1,
 {
          "indexmeta": "united states/country",[
            {
  "ontology_type": "country",             "relevancecontext": 0"First",
                "totalfrequencyfieldName": -1,"info",
                "typescript": "Country"var info        },
    = decode(text); info;",
   {             "actual_namescriptlang": "SCANNER_1javascript",
            "dimension": "What",}
             "disambiguated_name": "SCANNER_1"],
        "script": "function decode(x)\n{\n  "doccount": 0, var info = {};   \n    var rec "frequency": 1,= x.split(',');   \n    info.device = rec[0];\n    "index": "scanner_1/sensor",
  info.date = rec[1];\n    info.srcIP = rec[2];\n    "relevance": 0,
info.dstIP = rec[3];\n    info.alert = rec[4];\n    info.country  "totalfrequency": -1,= rec[5];\n    return info;\n}",
        "typesimpleTextCleanser": "Sensor"[
        },    {
    {             "actual_namefield": "DUMMY_ALERT_TYPE_1fullText",
                "dimensionflags": "Whatmd",
            "disambiguated_name    "replacement": "DUMMY_ALERT_TYPE_1 , ",
                "doccountscript": 0,",",
                "frequencyscriptlang": "regex"
1,             "index": "dummy_alert_type_1/alerttype",},
            {
  "relevance": 0,             "totalfrequencyfield": -1"description",
                "typeflags": "AlertTypemd",
        }     ],
    "fullTextreplacement": "SCANNER_1 , ", 2012-01-01T13:43:00 , 10.0.0.1 , 66.66.66.66 , DUMMY_ALERT_TYPE_1 , United States",
                "mediaTypescript": ["Log,"],,
                "metadatascriptlang": {"info": [{regex"
            }
        ]
    },
    "alertuseExtractor": "DUMMY_ALERT_TYPE_1 "none",
          "country": "United States",
        "date": "2012-01-01T13:43:00",
        "device": "SCANNER_1 ",
        "dstIP": "66.66.66.66",
        "srcIP": " 10.0.0.1"
    }]},
    "modified": "Jun 4, 2013 12:54:34 AM UTC",
    "publishedDate": "January 1, 2012 13:43:00 PM UTC",
    "source": ["Cyber Logs Test"],
    "sourceKey": ["INFINITE_ENDPOINT.api.share.get.51ad28a440b4a4f0f757824c.25.26"],
    "tags": [
        "cyber",
        "structured"
    ],
    "title": "DUMMY_ALERT_TYPE_1  @ 2012-01-01T13:43:00 [SCANNER_1 ]: 66.66.66.66 -> 10.0.0.1",
    "url": "http://INFINITE_ENDPOINT/api/share/get/51ad28a440b4a4f0f757824c#1"
}"useTextExtractor":"none"	
}

Note that the API key is not visible in any of the extracted documents (it is removed in the "searchConfig.script" code), and is also not visible in the source to anyone but the source owner and administrator (due to the "isPublic:false" field). As an alternative (from June 2013), a cookie can be used: (eg) "rss.httpFields": { "Cookie": "infinitecookie=api:API_KEY;" }.

Old format Source #2b - web (including uploaded fileshares), manual parsing using Java

As above, except "unstructuredAnalysis.script" will look like:

Code Block
var parser = new Packages.au.com.bytecode.opencsv.CSVParser();
function decode(x)
{
   var rec = parser.parseLine(x.toString());
   var info = {};   
   info.device = '' + rec[0];
   info.date = '' + rec[1];
   info.srcIP = '' + rec[2];
   info.dstIP = '' + rec[3];
   info.alert = '' + rec[4];
   info.country = '' + rec[5];
   return info;
}

(Note that the "'' +  <string-variable>" construct is necessary to convert from Java strings to javascript strings)

See the opencsv documentation for more details.