I am running Cribl version 4.12 and I am trying to collect Qualys detection logs using “qualys-detections” script by Jon Rust but it does not seems to be working.
My Qualys login is working on EU1 platform "https://qualysapi.qualys.eu", the required python3 and env are set
Here is the “qualys-detections” script
---------------------------------------------------------------------------------------------
#!/usr/bin/python3
import sys
import requests
import re
import os
from pprint import pprint
from time import time
from datetime import datetime
debug = True
action = ''
APIURL = "https://qualysapi.qualys.eu"
try:
API_USER = os.environ['CRIBL_username']
API_PWD = os.environ['CRIBL_password']
EARLIEST = os.environ['CRIBL_earliest']
MAX_RUN_COUNT = int(os.environ['CRIBL_max'])
except:
print("CRIBL_username, CRIBL_password, CRIBL_max and CRIBL_earliest must be defined environment vars")
sys.exit(1)
################################
# if the collect arg var exists, we already disco'd
if 'CRIBL_COLLECT_ARG' in os.environ:
ids = os.environ['CRIBL_COLLECT_ARG'].split(' ')
id_count = len(ids)
ids = ','.join(str(item) for item in ids)
action = 'collect'
else:
action = 'disco'
################################
def login():
url = APIURL + '/api/2.0/fo/session/'
payload = 'action=login&username='+API_USER+'&password='+API_PWD
length = len(payload)
headers = {
'X-Requested-With': 'Cribl Stream',
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': str(length)
}
response = requests.post(url, headers=headers, data=payload)
if response.status_code == 200:
return response.cookies['QualysSession']
else:
print('Error:', response.status_code)
sys.exit(1)
################################
def logout(cookie):
url = APIURL + '/api/2.0/fo/session/'
headers = {
'X-Requested-With': 'Cribl Stream',
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': '13',
'Cookie': 'QualysSession='+cookie,
'Cookie2': '$Version="1"'
}
payload = "action=logout"
response = requests.post(url, headers=headers, data=payload)
if response.status_code == 200:
return True
else:
print('Logout Error:', response.status_code)
pprint(vars(response.request))
sys.exit(1)
################################
def list_hosts(cookie):
url = APIURL + '/api/2.0/fo/asset/host/?action=list&truncation_limit=0&vm_processed_after=' + EARLIEST
headers = {
'X-Requested-With': 'Cribl Stream',
'Content-Type': 'application/x-www-form-urlencoded',
'Cookie': 'QualysSession='+cookie,
'Cookie2': '$Version="1"'
}
response = requests.get(url, headers=headers)
try:
if response.status_code == 200:
return(response.text)
else:
print('list_hosts failed with code:', response.status_code)
logout(cookie)
sys.exit(1)
except Exception as e:
print("list_hosts ERROR2: " + str(e))
print("logging out")
logout(cookie)
sys.exit()
################################
def run_detections(cookie):
url = APIURL + '/api/2.0/fo/asset/host/vm/detection/'
payload = 'output_format=XML&action=list&truncation_limit=0&detection_updated_since=' + EARLIEST + '&ids=' + ids
length = len(payload)
headers = {
'X-Requested-With': 'Cribl Stream',
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': str(length),
'Cookie': 'QualysSession='+cookie,
'Cookie2': '$Version="1"'
}
response = requests.post(url, headers=headers, data=payload)
#if debug:
# print("request")
# pprint(vars(response.request))
try:
if response.status_code == 200:
return(response.text)
else:
print('run_detection failed with code:', response.status_code)
#logout(cookie)
sys.exit(1)
except Exception as e:
print("run_detection ERROR: " + str(e))
sys.exit()
################################
# throw some debug info into a file
if debug:
output_file = '/tmp/qualys-debug-' + str(time()) + '.txt'
with open(output_file, 'w') as file:
if action == 'collect':
file.write('collect\ntimestamp: ' + EARLIEST)
else:
file.write('disco\ntimestamp: ' + EARLIEST)
# Dump environment variables
file.write('\nEnv:\n')
for key, value in os.environ.items():
if key.startswith('CRIBL'):
file.write(f'{key}: {value}\n')
################################
# first get logged in
cookie = login()
#if debug:
# print("got cookie %",cookie)
# get a list of ids if action is list
if action == 'disco':
# get the list of host ids
xml = list_hosts(cookie)
# print the list of host ids in MAX_RUN_COUNT chunks
try:
ids = re.findall(r'<ID>(.*?)</ID>', xml)
for i in range(0,len(ids),MAX_RUN_COUNT):
print(*ids[i:i+MAX_RUN_COUNT])
except Exception as e:
print("died at id extract")
pprint(vars(xml))
# or we want detections
elif action == 'collect':
try:
xml = run_detections(cookie)
print(xml)
except Exception as e:
print("died while getting detections")
pprint(vars(xml))
# debugging
if debug:
output_file = '/tmp/qualys-debug-out-' + str(time()) + '.txt'
with open(output_file, 'w') as file:
file.write(xml)
logout(cookie)
-------------------------------------------------------------------------------------------------------------------------------------
Error message I received but I am not sure it’s relating
ENOENT: no such file or directory, open '/opt/cribl/state/jobs/workergrp/1763055468.40061.adhoc.qualys-detections/status.json'"
message:
job: 1763055468.40061.adhoc.qualys-detections, no longer exists
Error
at new n (/opt/cribl/bin/cribl.js:15:117851)
at new P (/opt/cribl/bin/cribl.js:15:10031141)
at R.getTaskErrors (/opt/cribl/bin/cribl.js:15:10059453)
at async D.getTaskErrors (/opt/cribl/bin/cribl.js:15:11626441)
Can you please help troubleshoot the python3 code? the debug created in /tmp/qualys-debug-out.txt only shows the set variable
How do I run “qualys-detections.py” on command line while the python3 env variables are set?