How can be clean a nested python dicts structure without to know the structure and the sub object ty-CodePudding

In this example I want to delete the keys that includes nan values whithout to know the depth and the type of the objects.

{ 
  data:

        {   key1_level_1: "network", 
            key2_level_1: "proxy", 
            key3_level_1: "disabled", 
            key4_level_1: [
                {key1_level_2: "default", key2_level_2: "schema", key3_level_2: "ALL"}
            ], 
            key5_level_1: [
                {key4_level_2: nan, key5_level_2: nan}
            ], 
            key6_level_1: [
                {key6_level_2: "render", key6_level_2: "render2", key7_level_2: nan, 
                    key1_level_3: [
                            {key1_level_4: "192.168.1.0/24", key2_level_4: 
                                "public", key3_level_4: nan, key4_level_4: nan, key5_level_4: nan}
                        ]}
            ]
        }
 }

I start to create a base method but this idea will not working if the level of structure will be more complex.

def clean_null_items(self, data):

    if isinstance(data, dict): 
        for _, v in data.items():
            if isinstance(v, str):
                clean(str)
            if isinstance(v, list):
                self.clean_null_items(v)

    if isinstance(data, list):
        for v in data:
            if isinstance(v, list):
                self.clean_null_items(va) 
            else:
                clean(v) 
    return data

The result should be :

  {
    data :
        {   key1_level_1: "network", 
            key2_level_1: "proxy", 
            key3_level_1: "disabled", 
            key4_level_1: [
                {key1_level_2: "default", key2_level_2: "schema", key3_level_2: "ALL"}
            ], 
            key6_level_1: [
                {key6_level_2: "render", key6_level_2: "render2",
                    key1_level_3: [
                            {key1_level_4: "192.168.1.0/24", key2_level_4: "public" }
                        ]}
            ]
        }
   }

CodePudding user response：

I just defined the nan as a string, you can create a new method that does the checking for your data.

nan = "nan"
sample = [
  {
    "key1_level_1": "network",
    "key2_level_1": "proxy",
    "key3_level_1": "disabled",
    "key4_level_1": [
      {
        "key1_level_2": "default",
        "key2_level_2": "schema",
        "key3_level_2": "ALL"
      }
    ],
    "key5_level_1": [
      {
        "key4_level_2": nan,
        "key5_level_2": nan
      }
    ],
    "key6_level_1": [
      {
        "key6_level_2": "render2",
        "key7_level_2": nan,
        "key1_level_3": [
          {
            "key1_level_4": "192.168.1.0/24",
            "key2_level_4": "public",
            "key3_level_4": nan,
            "key4_level_4": nan,
            "key5_level_4": nan
          }
        ]
      }
    ]
  }
]

def clean_null_items(data):
    if isinstance(data, dict):
        for k in list(data.keys()):
            v = data[k]
            if isinstance(v, str) and v == nan:
                del data[k]
            if isinstance(v, list):
                clean_null_items(v)

    if isinstance(data, list):
        for v in data:
            clean_null_items(v)
    return data

from pprint import pprint
pprint(clean_null_items(sample))

The function modifies the data in place instead of creating a new copy like @SUTerliakov's answer.

Above outputs -

[{'key1_level_1': 'network',
  'key2_level_1': 'proxy',
  'key3_level_1': 'disabled',
  'key4_level_1': [{'key1_level_2': 'default',
                    'key2_level_2': 'schema',
                    'key3_level_2': 'ALL'}],
  'key5_level_1': [{}],
  'key6_level_1': [{'key1_level_3': [{'key1_level_4': '192.168.1.0/24',
                                      'key2_level_4': 'public'}],
                    'key6_level_2': 'render2'}]}]

CodePudding user response：

You need a technique called recursion. If value is a collection (dict or list), you can call method on each element (no matter is it collection or not) and combine results. If some of values were nested, they will work the same way and go deeper.

def isnan(val):
    ''' This should return True iff value is nan 
    
    I don't know what does your `nan` represent, so leaving it like this '''
    return val is None
    # return np.isnan(val)

def clean_null_items(self, data):
    if isinstance(data, dict):
        return {key: cleaned
                for key, val in data.items()
                if (cleaned := self.clean_null_items(val)) is not None}

    if isinstance(data, list):
        return [cleaned
                for val in data
                if (cleaned := self.clean_null_items(val)) is not None]

    return data if not isnan(data) else None

CodePudding user response：

The issue was that NaN or nan was a float not a str :(

Here the finale solution that working.

def clean_null_items(self, data):
    if isinstance(data, dict):
        for k in list(data.keys()):
            v = data[k]
            if isinstance(v, float):
                is_NaN = math.isnan(v)
                if is_NaN is True :
                    del data[k]
            if isinstance(v, list):
                self.clean_null_items(v)
    if isinstance(data, list):
        for v in data:
            self.clean_null_items(v)
    return data