{
  "35.0.0.0.0": {
    "hostnames": [
      {
        "name": "domain.com",
        "type": "PTR"
      }
    ],
    "addresses": {
      "ipv4": "35.0.0.0"
    },
    "vendor": {},
    "status": {
      "state": "up",
      "reason": "syn-ack"
    },
    "tcp": {
      "30000": {
        "state": "open",
        "reason": "syn-ack",
        "name": "http",
        "product": "nginx",
        "version": "1.20.0",
        "extrainfo": "",
        "conf": "10",
        "cpe": "cpe:/a:igor_sysoev:nginx:1.20.0",
        "script": {
          "http-server-header": "nginx/1.20.0",
          "vulners": "\n  cpe:/a:igor_sysoev:nginx:1.20.0: \n    \tNGINX:CVE-2021-23017\t6.8\thttps://vulners.com/nginx/NGINX:CVE-2021-23017\n    \t9A14990B-D52A-56B6-966C-6F35C8B8EB9D\t6.8\thttps://vulners.com/githubexploit/9A14990B-D52A-56B6-966C-6F35C8B8EB9D\t*EXPLOIT*\n    \t1337DAY-ID-36300\t6.8\thttps://vulners.com/zdt/1337DAY-ID-36300\t*EXPLOIT*\n    \tPACKETSTORM:162830\t0.0\thttps://vulners.com/packetstorm/PACKETSTORM:162830\t*EXPLOIT*"
        }
      }
    }
  }
}

I want to extract "vulners" value here i tried this -

def scandata():
results = []
for x in collection.find({},{"scan": 1, "_id": 0 }):
    results.append(json.loads(json_util.dumps(x)))


portnumber = []
datay = []


for result in results:
    ips = result['scan']


for ip in ips:

        ports = result['scan'][ip]['tcp']
        ipdomain = result['scan'][ip]['hostnames']

        for port in ports:
            portnumber.append(port)
            datax = ports[port]['script']['vulners']
            datay.append(datax)


        return ({'domain': ipdomain, 'ports': portnumber, 'product': datay, 'vulner': datax})

But it gives me key error, I tried looping over that still didn't work! What should i do to get that value? Also im using mongodb so is there anyway to grab it from there too? Please help!

CodePudding user response：

this approach is based on a model of your data you want to parse. From my point of view this is more work in the beginning. With the advantage, that you will have clean error messages and you can control the behaviour by adapting your data model.

1. make a model of the data you want to parse

from typing import Any, Optional
from pydantic import BaseModel, Field

class ExScript(BaseModel):
    vulners:str = ""

class Ex30000(BaseModel):
    script:ExScript = Field(default=Any)
        
class ExTcp(BaseModel):
    root:Ex30000= Field(default=Any, alias="30000")
    
class ExRoot(BaseModel):
    tcp:ExTcp = Field() # Required
    
class Base(BaseModel):
    root:ExRoot = Field(default=Any, alias="35.0.0.0.0")

2. change your input data to a raw string outherwise you will have to escape \n and \t

input_will_work = r"""{
  "35.0.0.0.0": {
    "hostnames": [
      {
        "name": "domain.com",
        "type": "PTR"
      }
    ],
    "addresses": {
      "ipv4": "35.0.0.0"
    },
    "vendor": {},
    "status": {
      "state": "up",
      "reason": "syn-ack"
    },
    "tcp": {
      "30000": {
        "state": "open",
        "reason": "syn-ack",
        "name": "http",
        "product": "nginx",
        "version": "1.20.0",
        "extrainfo": "",
        "conf": "10",
        "cpe": "cpe:/a:igor_sysoev:nginx:1.20.0",
        "script": {
          "http-server-header": "nginx/1.20.0",
          "vulners": "\n  cpe:/a:igor_sysoev:nginx:1.20.0: \n    \tNGINX:CVE-2021-23017\t6.8\thttps://vulners.com/nginx/NGINX:CVE-2021-23017\n    \t9A14990B-D52A-56B6-966C-6F35C8B8EB9D\t6.8\thttps://vulners.com/githubexploit/9A14990B-D52A-56B6-966C-6F35C8B8EB9D\t*EXPLOIT*\n    \t1337DAY-ID-36300\t6.8\thttps://vulners.com/zdt/1337DAY-ID-36300\t*EXPLOIT*\n    \tPACKETSTORM:162830\t0.0\thttps://vulners.com/packetstorm/PACKETSTORM:162830\t*EXPLOIT*"
        }
      }
    }
  }
}
"""

input_will_fail = r"""{
  "35.0.0.0.0": {}
}
"""

3.1 this should give you the expected result

obj1 = Base.parse_raw(input_will_work)
print(obj1.root.tcp.root.script.vulners)

3.2 this should throw an exception

obj2 = Base.parse_raw(input_will_fail)

CodePudding user response：

def extract_data(result):
    domains = result["hostnames"]
    ports_data = []
    # Each port can have different products and vulners
    # So that data is grouped together in a dictionary
    for port in result["tcp"].keys():
        port_data = result["tcp"][port]
        product = port_data["product"]
        vulners = port_data['script']['vulners']
        ports_data.append({
            "port":port,
            "product":product,
            "vulners":vulners
        })

    return {
        "domains":domains,
        "ports_data": ports_data
        }

Your Data source isn't clear, so gettting that from such a dictionary

{
  "35.0.0.0.0": {
    "hostnames": [
      {
        "name": "domain.com",
        "type": "PTR"
      }
    ],
    "addresses": {
      "ipv4": "35.0.0.0"
    },
    "vendor": {},
    "status": {
      "state": "up",
      "reason": "syn-ack"
    },
    "tcp": {
      "30000": {
        "state": "open",
        "reason": "syn-ack",
        "name": "http",
        "product": "nginx",
        "version": "1.20.0",
        "extrainfo": "",
        "conf": "10",
        "cpe": "cpe:/a:igor_sysoev:nginx:1.20.0",
        "script": {
          "http-server-header": "nginx/1.20.0",
          "vulners": "\n  cpe:/a:igor_sysoev:nginx:1.20.0: \n    \tNGINX:CVE-2021-23017\t6.8\thttps://vulners.com/nginx/NGINX:CVE-2021-23017\n    \t9A14990B-D52A-56B6-966C-6F35C8B8EB9D\t6.8\thttps://vulners.com/githubexploit/9A14990B-D52A-56B6-966C-6F35C8B8EB9D\t*EXPLOIT*\n    \t1337DAY-ID-36300\t6.8\thttps://vulners.com/zdt/1337DAY-ID-36300\t*EXPLOIT*\n    \tPACKETSTORM:162830\t0.0\thttps://vulners.com/packetstorm/PACKETSTORM:162830\t*EXPLOIT*"
        }
      }
    }
  }
}

Usage

def scandata():
    # for ip_address in collection.find({})["scan"]:
    for ip_address in DataSourceDictionary:
        ip_address_data = extract_data(
            DataSourceDictionary[ip_address])
        )

        print(ip_address, ip_address_data)

Also, mongodb results can be used like normal dictionaries so no need to convert to json.