DC crime data source gallery

Sample document

nidccnreportdatetimeshiftoffensemethodblocksiteaddresslatitudelongitudecitystatewardancsmddistrictpsa
95577811012669Jan 29, 2011 12:00:00 AM UTCUNKTHEFT21300 B/O CONNECTICUT AVE NW38.90992780287290-77.04360677659660WASHINGTONDC22B2B02SECOND208.0

Source

{
    "description": "Incident reports from the Washington, D.C. MPD",
    "harvest": {
        "doccount": 9314,
        "harvest_status": "success,
        "harvested": "Nov 2, 2012 11:26:25 AM",
        "synced": "Nov 2, 2012 11:24:30 AM"
    },
    "isApproved": true,
    "isPublic": true,
    "mediaType": "Record",
    "tags": [
        "crime report",
        "MPD",
        "Washington, D.C."
    ],
    "title": "data.dc.gov - Crime Incidents (ASP)",
    "processingPipeline": [
        {
            "database": {
			    "authentication": {
			        "password": "PASSWORD",
			        "username": "USERNAME"
			    },
                "databaseName": "washingtondc",
                "databaseType": "mysql",
                "deleteQuery": "",
                "deltaQuery": "SELECT * FROM IncidentReport WHERE REPORTDATETIME >= (SELECT ADDDATE(CURDATE(),-7))",
                "hostname": "dbserver.ikanow.com",
                "port": "3306",
                "primaryKey": "NID",
                "publishedDate": "REPORTDATETIME",
                "query": "SELECT * FROM IncidentReport",
                "snippet": "OFFENSE",
                "title": "CCN",
                "url": "jdbc:mysql://ec2-184-72-206-97.compute-1.amazonaws.com/washingtondc/IncidentReport"
            }
        },
        {
            "docMetadata": {
                "description": "$metadata.reportdatetime: $metadata.offense,$metadata.method was reported at: $metadata.blocksiteaddress",
                "geotag": {
                    "lat": "$metadata.latitude",
                    "lon": "$metadata.longitude"
                }
            }
        },
        {
            "textEngine": {
                "engineName": "default"
            }
        },
        {
            "entities": [
                {
                    "dimension": "What",
                    "disambiguated_name": "$metadata.offense,$metadata.method",
                    "type": "CriminalActivity"
                },
                {
                    "dimension": "Where",
                    "disambiguated_name": "$metadata.blocksiteaddress,$metadata.city,$metadata.state",
                    "geotag": {},
                    "type": "Place"
                },
                {
                    "dimension": "Where",
                    "disambiguated_name": "$metadata.ward,$metadata.city,$metadata.state",
                    "type": "Place"
                },
                {
                    "dimension": "Where",
                    "disambiguated_name": "$metadata.district,$metadata.city,$metadata.state",
                    "type": "Place"
                }
            ]
        },
        {
            "associations": [
                {
                    "entity1": "$metadata.offense,$metadata.method",
                    "geotag": {},
                    "time_start": "$metadata.reportdatetime",
                    "verb": "reported",
                    "verb_category": "crime"
                }
            ]
        }
    ]
}

Sample output

{
    "_id": "4eb1a6532285a1eed2af759c",
    "title": "11012669",
    "url": "jdbc:mysql://dbserver.ikanow.com/washingtondc/IncidentReport/955778",
    "created": "Nov 2, 2011 08:21:39 PM UTC",
    "modified": "Nov 2, 2011 08:21:39 PM UTC",
    "publishedDate": "Jan 29, 2011 05:00:00 AM UTC",
    "source": [
        "data.dc.gov - Crime Incidents (ASP)"
    ],
    "sourceKey": [
        "Bergen.washingtondc.IncidentReport"
    ],
    "mediaType": [
        "Record"
    ],
    "description": "Jan 29, 2011 12:00:00 AM: THEFT,2 was reported at: 1300 B/O CONNECTICUT AVE NW",
    "entities": [
        {
            "disambiguated_name": "THEFT,2",
            "index": "theft,2/criminalactivity",
            "actual_name": "THEFT,2",
            "type": "CriminalActivity",
            "frequency": 1,
            "totalfrequency": 2705,
            "doccount": 2705,
            "dimension": "What"
        },
        {
            "disambiguated_name": "1300 B/O CONNECTICUT AVE NW,WASHINGTON,DC",
            "index": "1300 b/o connecticut ave nw,washington,dc/place",
            "actual_name": "1300 B/O CONNECTICUT AVE NW,WASHINGTON,DC",
            "type": "Place",
            "frequency": 1,
            "totalfrequency": 28,
            "doccount": 28,
            "geotag": {
                "lat": 38.9099278028729,
                "lon": -77.0436067765966
            },
            "dimension": "Where",
            "ontology_type": "point"
        },
        {
            "disambiguated_name": "2,WASHINGTON,DC",
            "index": "2,washington,dc/place",
            "actual_name": "2,WASHINGTON,DC",
            "type": "Place",
            "frequency": 1,
            "totalfrequency": 2326,
            "doccount": 2326,
            "dimension": "Where"
        },
        {
            "disambiguated_name": "SECOND,WASHINGTON,DC",
            "index": "second,washington,dc/place",
            "actual_name": "SECOND,WASHINGTON,DC",
            "type": "Place",
            "frequency": 1,
            "totalfrequency": 2120,
            "doccount": 2120,
            "dimension": "Where"
        }
    ],
    "tags": [
        "crime report",
        "MPD",
        "Washington, D.C."
    ],
    "communityId": [
        "4c927585d591d31d7c37097b"
    ],
    "associations": [
        {
            "entity1": "theft,2",
            "entity1_index": "theft,2/criminalactivity",
            "verb": "reported",
            "verb_category": "crime",
            "time_start": "2011-01-29T00:00:00",
            "geotag": {
                "lat": 38.9099278028729,
                "lon": -77.0436067765966
            },
            "assoc_type": "Summary"
        }
    ],
    "metadata": {
        "nid": [
            [
                {
                    "value": "955778.0"
                }
            ]
        ],
        "ccn": [
            [
                {
                    "value": "1.1012669E7"
                }
            ]
        ],
        "reportdatetime": [
            [
                "Jan 29, 2011 12:00:00 AM UTC"
            ]
        ],
        "shift": [
            [
                "UNK"
            ]
        ],
        "offense": [
            [
                "THEFT"
            ]
        ],
        "method": [
            [
                "2"
            ]
        ],
        "blocksiteaddress": [
            [
                "1300 B/O CONNECTICUT AVE NW"
            ]
        ],
        "latitude": [
            [
                "38.90992780287290"
            ]
        ],
        "longitude": [
            [
                "-77.04360677659660"
            ]
        ],
        "city": [
            [
                "WASHINGTON"
            ]
        ],
        "state": [
            [
                "DC"
            ]
        ],
        "ward": [
            [
                {
                    "value": "2.0"
                }
            ]
        ],
        "anc": [
            [
                "2B"
            ]
        ],
        "smd": [
            [
                "2B02"
            ]
        ],
        "district": [
            [
                "SECOND"
            ]
        ],
        "psa": [
            [
                {
                    "value": "208.0"
                }
            ]
        ]
    },
    "docGeo": {
        "lat": 38.9099278028729,
        "lon": -77.0436067765966
    }
}

Annex - old format Source

{
    "authentication": {
        "password": "PASSWORD",
        "username": "USERNAME"
    },
    "database": {
        "databaseName": "washingtondc",
        "databaseType": "mysql",
        "deleteQuery": "",
        "deltaQuery": "SELECT * FROM IncidentReport WHERE REPORTDATETIME >= (SELECT ADDDATE(CURDATE(),-7))",
        "hostname": "dbserver.ikanow.com",
        "port": "3306",
        "primaryKey": "NID",
        "publishedDate": "REPORTDATETIME",
        "query": "SELECT * FROM IncidentReport",
        "snippet": "OFFENSE",
        "title": "CCN"
    },
    "description": "Incident reports from the Washington, D.C. MPD",
    "extractType": "Database",
    "harvest": {
        "doccount": 9314,
        "harvest_message": "Error when harvesting DB: Cannot establish connection to the database : jdbc:mysql://ec2-184-72-206-97.compute-1.amazonaws.com:3306/washingtondc",
        "harvest_status": "error",
        "harvested": "Nov 2, 2012 11:26:25 AM",
        "synced": "Nov 2, 2012 11:24:30 AM"
    },
    "isApproved": true,
    "isPublic": true,
    "mediaType": "Record",
    "structuredAnalysis": {
        "associations": [{
            "entity1": "$metadata.offense,$metadata.method",
            "geotag": {},
            "time_start": "$metadata.reportdatetime",
            "verb": "reported",
            "verb_category": "crime"
        }],
        "description": "$metadata.reportdatetime: $metadata.offense,$metadata.method was reported at: $metadata.blocksiteaddress",
        "docGeo": {
            "lat": "$metadata.latitude",
            "lon": "$metadata.longitude"
        },
        "entities": [
            {
                "dimension": "What",
                "disambiguated_name": "$metadata.offense,$metadata.method",
                "type": "CriminalActivity"
            },
            {
                "dimension": "Where",
                "disambiguated_name": "$metadata.blocksiteaddress,$metadata.city,$metadata.state",
                "geotag": {},
                "type": "Place"
            },
            {
                "dimension": "Where",
                "disambiguated_name": "$metadata.ward,$metadata.city,$metadata.state",
                "type": "Place"
            },
            {
                "dimension": "Where",
                "disambiguated_name": "$metadata.district,$metadata.city,$metadata.state",
                "type": "Place"
            }
        ]
    },
    "tags": [
        "crime report",
        "MPD",
        "Washington, D.C."
    ],
    "title": "data.dc.gov - Crime Incidents (ASP)",
    "url": "jdbc:mysql://ec2-184-72-206-97.compute-1.amazonaws.com/washingtondc/IncidentReport",
    "useExtractor": "none"
}