Extract data from json, append using for loop and save as CSV-CodePudding

I have extracted id, username, and name for 100 followers for 102 politicians using Tweepy. The data is stored in a JSON file named pol_followers. Now I wish to append id and username and save it as a CSV file using the function below. However, when using the function in the last line append_followers_to_csv(pol_followers, "pol_followers.csv") I get the error seen at the bottom.

# Structure of pol_followers. The full pol_followers is much longer... 
print(json.dumps(pol_followers, indent=4, sort_keys=True))  # see json data structure

[
{
        "data": [
            {
                "id": "1464206217807601666",
                "name": "terry alex",
                "username": "terryal51850644"
            },
            {
                "id": "1479032154394968064",
                "name": "Charles Williams",
                "username": "Charles99924770"
            },
            {
                "id": "2526015770",
                "name": "LISA P",
                "username": "LISAP0910"
            },
            {
                "id": "2957692520",
                "name": "fayaz ahmad",
                "username": "ahmadfayaz202"
            }
        ],
        "meta": {
            "next_token": "F6HS7IU5SRGHEZZZ",
            "result_count": 100
        }
    },
    {
        "data": [
            {
                "id": "2482703136",
                "name": "HieuVu",
                "username": "sachieuhaihanh"
            },
            {
                "id": "580882148",
                "name": "Maxine D. Harmon",
                "username": "maxxximd"
            },
            {
                "id": "1478867472841334787",
                "name": "RBPsych1",
                "username": "RBPsych1"

# Create file
csv_follower_file = open("pol_followers.csv", "a", newline="", encoding='utf-8')
csv_follower_writer = csv.writer(csv_follower_file)

# Create headers for the data I want to save. I only want to save these columns in my dataset
csv_follower_writer.writerow(
    ['id', 'username'])
csv_follower_file.close()\


def append_followers_to_csv(pol_followers, csv_follower_file):
    # A counter variable
    global follower_id, username
    counter = 0

    # Open OR create the target CSV file
    csv_follower_file = open(csv_follower_file, "a", newline="", encoding='utf-8')
    csv_follower_writer = csv.writer(csv_follower_file)

    for ids in pol_followers['data']:

        # 1. follower ID
        follower_id = ids['id']

        # 2. follower username
        username = ids['username']

        # Assemble all data in a list
        ress = [follower_id, username]

        # Append the result to the CSV file
        csv_follower_writer.writerow(ress)
        counter  = 1

        # When done, close the CSV file
        csvFile.close()

    # Print the number of tweets for this iteration
    print("# of Tweets added from this response: ", counter)


append_followers_to_csv(pol_followers, "pol_followers.csv")  # Save tweet data in a csv file

 File "<input>", line 1, in <module>
  File "<input>", line 11, in append_followers_to_csv
TypeError: list indices must be integers or slices, not str

CodePudding user response：

You are just missing additional loop, like so:

for each_dict in pol_followers:
    for ids in each_dict['data']:
        follower_id = ids['id']
        username = ids['username']

CodePudding user response：

You seem to have wrapped your JSON object in a list, so instead of getting the 'data' bit of the JSON, you are getting the 'data'th element of a list when you are iterating in your append_followers_to_csv function, which you can't do in python. Try removing the square brackets around the JSON or making it for ids in pol_followers[0]['data'].