Using the Datalake for EDR

Understanding the Datalake

The Armor data lake is a centralized repository for storing Armor collected data. With regards to EDR, the data lake contains data for incidents in every environment, including endpoints. This can be a lot of data so narrowing down the scope of information is critical to making sense of it all.

Accessing the Datalake

In the Armor Management Portal (AMP), navigate to Security -> Log Search and SSO into ChaosSearch.
Create a filter by doing the following:
1. Click on Add filter.
2. In Field select index.type
3. Select is for Operator.
4. In the Value field, type "endpoint-detections."
5. Click Save.
Now set the date range to encompass the incident date or dates to show and click Refresh.

Data Presentation

Data consists of documents stored in the datalake. Each document contains all the data related to that particular rule and resource. Below are examples of the table and JSON views:

Table Example

Field	Value
@timestamp	Feb 24, 2021 @ 17:44:24.340
@version	1
_id	753702.0
_index	1024_5595_customer
_score	1
_type	doc
armor_metadata.customer.account_name	Sales Demo_Anywhere_SE
armor_metadata.customer.hostname	C02WC4PPHTD5
armor_metadata.customer.os_name	macOS Catalina
armor_metadata.customer.product_name	AA
armor_metadata.customer.service_provider	Armor Anywhere
armor_metadata.customer.tenant_id	5595
armor_metrics.input_port	5443
armor_metrics.latency.processing	2.54
armor_metrics.processing_chain	["KVN_V4_collector_i-0ff8e8423488756d3\|2021-02-24T23:44:24Z","KVN_V4_processor_i-00e1d66f921030cf3\|2021-02-24T23:44:26Z"]
cs_partition_key_0	005595
data_type	armor-security-logs
document_size	4,480
event.action	ACTION_CREATE_PROCESS
event.id	c7c956f076f811eb977689669fc3b6cd
event.provider	NGAV
event.timezone	UTC
event.type	endpoint.event.procstart
event_uuid	adca21d5-803a-4063-929d-2298f9efcc7f
external_id	d2e4fdff-8743-4d6b-80fc-3f193d3974e2
host.hostname	C02WC4PPHTD5
host.id	37305327
host.os.name	MAC
hostname	C02WC4PPHTD5
index_type	endpoint-detections
labels.parent_id	1024
logsource.origin	unknown
message_size	1,922
network.direction	unknown
network.type	endpoint.event.procstart
organization.id	N88FDVZL
original_timestamp	Feb 24, 2021 @ 17:44:24.296
process.command_line	jamf policy -randomDelaySeconds 300
process.executable	/usr/local/jamf/bin/jamf
process.guid	N88FDVZL-02393bef-0000c380-00000000-1d70b0551f56e20
process.hash.md5	5b9533eacd04697f21f80eef3ba91377
process.hash.sha256	1984435bf0a3020af49c152776c3ad3a5a5aa6dc30b7b6ea08ab683da4a5d61b
process.parent.command_line	xpcproxy com.jamfsoftware.task.Every 15 Minutes
process.parent.executable	/usr/libexec/xpcproxy
process.parent.guid	N88FDVZL-02393bef-0000c380-00000000-1d70b0551f435a0
process.parent.hash.sha256	87477a57c83ce40d53ae865d806f30d437c0b0eba37db244014319db2fb1a934
process.parent.pid	50048
process.pid	50048
process.reputation	REP_NOT_LISTED
process.terminated	false
process.username	root
received_timestamp	Feb 24, 2021 @ 17:44:24.340
tags	["customer","confirmed_external_id"]
tenant_id	5595
threat.framework	[]
type	carbon-black

JSON Example

{
  "_score": 1,
  "_type": "doc",
  "_source": {
    "process.hash.md5": "5b9533eacd04697f21f80eef3ba91377",
    "document_size": 4480,
    "@timestamp": "2021-02-24T23:44:24.340Z",
    "event.provider": "NGAV",
    "tenant_id": "5595",
    "process.parent.pid": "50048",
    "network.type": "endpoint.event.procstart",
    "armor_metadata.customer.tenant_id": "5595",
    "hostname": "C02WC4PPHTD5",
    "host.os.name": "MAC",
    "message_size": 1922,
    "process.parent.executable": "/usr/libexec/xpcproxy",
    "_id": 753702,
    "tags": "[\"customer\",\"confirmed_external_id\"]",
    "armor_metrics.processing_chain": "[\"KVN_V4_collector_i-0ff8e8423488756d3|2021-02-24T23:44:24Z\",\"KVN_V4_processor_i-00e1d66f921030cf3|2021-02-24T23:44:26Z\"]",
    "armor_metadata.customer.hostname": "C02WC4PPHTD5",
    "event.id": "c7c956f076f811eb977689669fc3b6cd",
    "host.id": "37305327",
    "armor_metrics.input_port": "5443",
    "process.reputation": "REP_NOT_LISTED",
    "original_timestamp": "2021-02-24T23:44:24.296Z",
    "logsource.origin": "unknown",
    "process.terminated": "false",
    "process.guid": "N88FDVZL-02393bef-0000c380-00000000-1d70b0551f56e20",
    "event.timezone": "UTC",
    "process.parent.command_line": "xpcproxy com.jamfsoftware.task.Every 15 Minutes",
    "process.hash.sha256": "1984435bf0a3020af49c152776c3ad3a5a5aa6dc30b7b6ea08ab683da4a5d61b",
    "process.command_line": "jamf policy -randomDelaySeconds 300",
    "network.direction": "unknown",
    "received_timestamp": "2021-02-24T23:44:24.340Z",
    "process.parent.guid": "N88FDVZL-02393bef-0000c380-00000000-1d70b0551f435a0",
    "data_type": "armor-security-logs",
    "armor_metadata.customer.account_name": "Sales Demo_Anywhere_SE",
    "event_uuid": "adca21d5-803a-4063-929d-2298f9efcc7f",
    "organization.id": "N88FDVZL",
    "process.executable": "/usr/local/jamf/bin/jamf",
    "labels.parent_id": "1024",
    "armor_metadata.customer.service_provider": "Armor Anywhere",
    "process.parent.hash.sha256": "87477a57c83ce40d53ae865d806f30d437c0b0eba37db244014319db2fb1a934",
    "external_id": "d2e4fdff-8743-4d6b-80fc-3f193d3974e2",
    "armor_metrics.latency.processing": 2.5396230220794678,
    "process.username": "root",
    "cs_partition_key_0": "005595",
    "type": "carbon-black",
    "armor_metadata.customer.product_name": "AA",
    "event.type": "endpoint.event.procstart",
    "armor_metadata.customer.os_name": "macOS Catalina",
    "@version": 1,
    "host.hostname": "C02WC4PPHTD5",
    "event.action": "ACTION_CREATE_PROCESS",
    "threat.framework": "[]",
    "index_type": "endpoint-detections",
    "process.pid": "50048"
  },
  "_id": "753702.0",
  "_index": "1024_5595_customer"
}