Cortex Data Lake Quickstart

note

The following guide will walk you through making your first API request to the Cortex Data Lake.

Choose a Language

Python NodeJS Java

SDK Installation

Python

pip install pan-cortex-data-lake

NodeJS

npm i @paloaltonetworks/pan-cortex-data-lake

Java

Pre-compiled binaries are available in the /target folder of the GitHub repo. Check sha512 signature before trusting pre-build binaries.

Authentication

Perhaps the easiest way to get started is by leveraging a Developer Token provided by the API Explorer's token redemption service. See Developer Tokens for details.

Export environment variables:

Python

export PAN_DEVELOPER_TOKEN=<your_developer_token>export PAN_DEVELOPER_TOKEN_PROVIDER=https://app.apiexplorer.rocks/request_token

NodeJS

export PAN_DEVELOPER_TOKEN=<your_developer_token>export PAN_DEVELOPER_TOKEN_PROVIDER=https://app.apiexplorer.rocks/request_token

Java

export PAN_DEVELOPER_TOKEN=<your_developer_token>export PAN_DEVELOPER_TOKEN_PROVIDER=https://app.apiexplorer.rocks/request_token

Instantiate credentials object

Python

from pan_cortex_data_lake import Credentials

c = Credentials()

NodeJS

const hub = require("@paloaltonetworks/pan-cortex-hub");const cred = hub.DevTokenCredentials.factory();

Java

import com.paloaltonetworks.cortex.hub.HubCredentialsDevToken
var cred = HubCredentialsDevToken.factory();

Verify credentials

Next, let's test to make sure the credentials object is working as expected:

Python

c.refresh()

Example output:

>>> c.refresh()'eyJ...<your_access_token>'

NodeJS

cred.getToken().then(console.log);

Java

System.out.println(cred.apply(true));

Basic usage

Now that your credentials are set, let's make our first API request!

1. Import the package:

Python

from pan_cortex_data_lake import QueryService

NodeJS

const dl = require("@paloaltonetworks/pan-cortex-data-lake");

Java

import com.paloaltonetworks.cortex.data_lake.QueryServiceClient

2. Construct a Query Service object

Python

qs = QueryService(credentials=c)

NodeJS

const qsc = dl.QueryServiceClient.factory({ cortexDefCredentials: cred });

Java

var qsc = new QueryServiceClient(cred);

3. Define a SQL statement

Python

SQL = "SELECT source_ip, dest_ip from `<tenant_id>.firewall.traffic` LIMIT 5"

NodeJS

const sqlCmd = "SELECT source_ip, dest_ip from `<tenant_id>.firewall.traffic` LIMIT 5";

Java

var sqlCmd = "SELECT source_ip, dest_ip from `<tenant_id>.firewall.traffic` LIMIT 5";

4. Perform the query

Python

q = qs.create_query(query_params={"query": SQL})

NodeJS

const iter = qsc.iterator(sqlCmd);

Java

var iter = qsc.iterable(sqlCmd);

5. Print the output

Python

job_id = q.json()["jobId"]
for p in qs.iter_job_results(job_id=job_id):    print(p.text)

NodeJS

(async () => {  for await (const page of iter2) console.log(page);})();

Java

for (var item : iter) System.out.println(item);

Example output:

Python

(output formatted for display purposes)

{  "jobId": "9c276960-d0ef-49e5-8da8-31aaab27ee96",  "state": "DONE",  "rowsInJob": 5,  "rowsInPage": 5,  "page": {    "pageCursor": null,    "result": {      "data": [        {          "source_ip": {            "value": "10.154.1.20",            "hex": "00000000000000000000ffff0a9a0114"          },          "dest_ip": {            "value": "212.180.157.132",            "hex": "00000000000000000000ffffd4b49d84"          }        },        {          "source_ip": {            "value": "10.154.1.20",            "hex": "00000000000000000000ffff0a9a0114"          },          "dest_ip": {            "value": "212.180.157.132",            "hex": "00000000000000000000ffffd4b49d84"          }        },        {          "source_ip": {            "value": "10.154.1.20",            "hex": "00000000000000000000ffff0a9a0114"          },          "dest_ip": {            "value": "212.180.157.132",            "hex": "00000000000000000000ffffd4b49d84"          }        },        {          "source_ip": {            "value": "10.154.1.20",            "hex": "00000000000000000000ffff0a9a0114"          },          "dest_ip": {            "value": "212.180.157.132",            "hex": "00000000000000000000ffffd4b49d84"          }        },        {          "source_ip": {            "value": "10.154.1.20",            "hex": "00000000000000000000ffff0a9a0114"          },          "dest_ip": {            "value": "212.180.157.132",            "hex": "00000000000000000000ffffd4b49d84"          }        }      ]    }  },  "resultFormat": "valuesDictionary"}

NodeJS

2/25/2020, 13:17:28 CORTEX_SDK initial autorization header for default data lake2/25/2020, 13:17:29 CORTEX_SDK Created new HTTP2 session to cortex-prd1-api.us.cdl.paloaltonetworks.com[  {    source_ip: {      value: '192.168.110.131',      hex: '00000000000000000000ffffc0a86e83'    },    dest_ip: { value: '70.48.1.139', hex: '00000000000000000000ffff4630018b' }  },  {    source_ip: { value: '10.154.1.5', hex: '00000000000000000000ffff0a9a0105' },    dest_ip: { value: '124.43.145.45', hex: '00000000000000000000ffff7c2b912d' }  },  {    source_ip: {      value: '10.154.246.167',      hex: '00000000000000000000ffff0a9af6a7'    },    dest_ip: { value: '69.63.176.188', hex: '00000000000000000000ffff453fb0bc' }  },  {    source_ip: { value: '10.154.9.40', hex: '00000000000000000000ffff0a9a0928' },    dest_ip: {      value: '123.193.27.118',      hex: '00000000000000000000ffff7bc11b76'    }  },  {    source_ip: {      value: '10.154.196.169',      hex: '00000000000000000000ffff0a9ac4a9'    },    dest_ip: {      value: '121.243.224.142',      hex: '00000000000000000000ffff79f3e08e'    }  }]

Java

INFO: Updated authentication header for default data lakepaloaltonetworks.com{"source_ip":{"value":"10.154.3.55","hex":"00000000000000000000ffff0a9a0337"},"dest_ip":{"value":"174.137.113.120","hex":"00000000000000000000ffffae897178"}}{"source_ip":{"value":"10.154.1.46","hex":"00000000000000000000ffff0a9a012e"},"dest_ip":{"value":"58.19.16.252","hex":"00000000000000000000ffff3a1310fc"}}{"source_ip":{"value":"10.154.1.46","hex":"00000000000000000000ffff0a9a012e"},"dest_ip":{"value":"58.19.16.252","hex":"00000000000000000000ffff3a1310fc"}}{"source_ip":{"value":"10.154.1.46","hex":"00000000000000000000ffff0a9a012e"},"dest_ip":{"value":"58.19.16.252","hex":"00000000000000000000ffff3a1310fc"}}{"source_ip":{"value":"10.154.3.96","hex":"00000000000000000000ffff0a9a0360"},"dest_ip":{"value":"123.138.238.43","hex":"00000000000000000000ffff7b8aee2b"}}

Complete Example

(Previous code snippets in a single block)

Python

from pan_cortex_data_lake import Credentials, QueryService

c = Credentials()qs = QueryService(credentials=c)
SQL = "SELECT source_ip, dest_ip from `<tenant_id>.firewall.traffic` LIMIT 5"q = qs.create_query(query_params={"query": SQL})job_id = q.json()["jobId"]
for p in qs.iter_job_results(job_id=job_id):    print(p.text)

NodeJS

const dl = require("@paloaltonetworks/pan-cortex-data-lake");const hub = require("@paloaltonetworks/pan-cortex-hub");const cred = hub.DevTokenCredentials.factory();const sqlCmd =  "SELECT source_ip, dest_ip from `<tenant_id>.firewall.traffic` LIMIT 5";
async function worker(iter) {  for await (const page of iter) console.log(page);}
const qsc = dl.QueryServiceClient.factory({ cortexDefCredentials: cred });worker(qsc.iterator(sqlCmd));

Java

import com.paloaltonetworks.cortex.data_lake.QueryServiceClientimport com.paloaltonetworks.cortex.hub.HubCredentialsDevToken
var cred = HubCredentialsDevToken.factory();var sqlCmd = "SELECT source_ip, dest_ip from `<tenant_id>.firewall.traffic` LIMIT 5";var qsc = new QueryServiceClient(cred);
for (var item : qsc.iterable(sqlCmd)) System.out.println(item);