I wrote a script which will basically import data from a json file and store it in the database of my Django Application. I'm simply pointing to a json file and i'm adding "facilities" if they don't exist or they get updated if the last modified date changes. It worked perfectly fine until i couldn't import the json file locally anymore and made some smaller changes to use an external json file. When i run the importer now it will only import the first two facilities and then tell me that all others are up to date even though they don't exist. I even modified and tested the json file manually to make sure it is not caused by a bug inside the json file. I will add the old code plus the modified code below. One of the main differences is that this part is now at the very bottom in the new version after the if statement:
for key, data_object in data.items():
And also that i'm using "import requests" in the new version with the following code at the bottom of the file. My feeling is that i made a mistake right there:
def handle(self, *args, **options):
"""
Call the function to import data from json url
"""
headers = {'Content-Type': 'application/json'}
response = requests.get(
url=IMPORT_URL,
headers=headers,
)
data = response.json()
for key, data_object in data.items():
self.import_facility_from_file(data_object)
New Version:
import requests
import json
from leads.models import Facility, FacilityAddress, FacilityInspectionInfo, FacilityComplaints
from django.core.management.base import BaseCommand
IMPORT_URL = 'https://importerdomain.test/test.json'
class Command(BaseCommand):
def import_facility_from_file(self, data):
UUID = data.get('UUID', None)
Name = data.get('Name', None)
IssuedNumber = data.get('IssuedNumber', None)
Capacity = data.get('Capacity', None)
Licensee = data.get('Licensee', None)
Email = data.get('Email', None)
AdministratorName = data.get('AdministratorName', None)
TelephoneNumber = data.get('TelephoneNumber', None)
ClosedTimestamp = data.get('ClosedTimestamp', None)
MostRecentLicenseTimestamp = data.get('MostRecentLicenseTimestamp', None)
ImporterLastModifiedTimestamp = data.get('ImporterLastModifiedTimestamp', None)
PrimaryAddress = data["AddressInfo"]["PrimaryAddress"]
SecondaryAddress = data["AddressInfo"]["SecondaryAddress"]
City = data["AddressInfo"]["City"]
RegionOrState = data["AddressInfo"]["RegionOrState"]
PostalCode = data["AddressInfo"]["PostalCode"]
Geolocation = data["AddressInfo"]["Geolocation"]
ComplaintRelatedVisits = data["InspectionInfo"]["ComplaintRelatedVisits"]
InspectionRelatedVisits = data["InspectionInfo"]["InspectionRelatedVisits"]
NumberOfVisits = data["InspectionInfo"]["NumberOfVisits"]
LastVisitTimestamp = data["InspectionInfo"]["LastVisitTimestamp"]
ComplaintsTypeA = data["Complaints"]["ComplaintsTypeA"]
ComplaintsTypeB = data["Complaints"]["ComplaintsTypeB"]
SubstantiatedAllegations = data["Complaints"]["SubstantiatedAllegations"]
TotalAllegations = data["Complaints"]["TotalAllegations"]
if Facility.objects.filter(ImporterLastModifiedTimestamp=ImporterLastModifiedTimestamp):
msg = "\n\nfacility exists and is up to date: {}\n{}".format(Name, str())
print(msg)
else:
print("UPDATE")
facility, facility_created = Facility.objects.update_or_create(UUID=UUID,
defaults={
'Name': Name,
'IssuedNumber': IssuedNumber,
'Capacity': Capacity,
'Licensee': Licensee,
'Email': Email,
'AdministratorName': AdministratorName,
'TelephoneNumber': TelephoneNumber,
'ClosedTimestamp': ClosedTimestamp,
'MostRecentLicenseTimestamp': MostRecentLicenseTimestamp,
'ImporterLastModifiedTimestamp': ImporterLastModifiedTimestamp
}
)
facility_address, address_created = FacilityAddress.objects.update_or_create(AddressInfo=facility,
defaults={
'PrimaryAddress': PrimaryAddress,
'SecondaryAddress': SecondaryAddress,
'City': City,
'RegionOrState': RegionOrState,
'PostalCode': PostalCode,
'Geolocation': Geolocation,
'AddressInfo': facility
}
)
facility_inspection, inspection_created = FacilityInspectionInfo.objects.update_or_create(InspectionInfo=facility,
defaults={
'ComplaintRelatedVisits': ComplaintRelatedVisits,
'InspectionRelatedVisits': InspectionRelatedVisits,
'NumberOfVisits': NumberOfVisits,
'LastVisitTimestamp': LastVisitTimestamp,
'InspectionInfo': facility
}
)
facility_complaints, complaints_created = FacilityComplaints.objects.update_or_create(Complaints=facility,
defaults={
'ComplaintsTypeA': ComplaintsTypeA,
'ComplaintsTypeB': ComplaintsTypeB,
'SubstantiatedAllegations': SubstantiatedAllegations,
'TotalAllegations': TotalAllegations,
'Complaints': facility
}
)
def handle(self, *args, **options):
"""
Call the function to import data from json url
"""
headers = {'Content-Type': 'application/json'}
response = requests.get(
url=IMPORT_URL,
headers=headers,
)
data = response.json()
for key, data_object in data.items():
self.import_facility_from_file(data_object)
Old Version
import os
import json
import traceback
from data_import.models import Facility, FacilityAddress, FacilityInspectionInfo, FacilityComplaints
from django.core.management.base import BaseCommand
from datetime import datetime
from jsontest.settings import BASE_DIR, STATIC_URL
class Command(BaseCommand):
def import_facility_from_file(self):
print("BASE", BASE_DIR)
data_folder = os.path.join(BASE_DIR, 'import_data', 'resources')
for data_file in os.listdir(data_folder):
with open(os.path.join(data_folder, data_file), encoding='utf-8') as data_file:
data = json.loads(data_file.read())
for key, data_object in data.items():
UUID = data_object.get('UUID', None)
Name = data_object.get('Name', None)
IssuedNumber = data_object.get('IssuedNumber', None)
Capacity = data_object.get('Capacity', None)
Licensee = data_object.get('Licensee', None)
Email = data_object.get('Email', None)
AdministratorName = data_object.get('AdministratorName', None)
TelephoneNumber = data_object.get('TelephoneNumber', None)
ClosedTimestamp = data_object.get('ClosedTimestamp', None)
MostRecentLicenseTimestamp = data_object.get('MostRecentLicenseTimestamp', None)
PrimaryAddress = data_object["AddressInfo"]["PrimaryAddress"]
SecondaryAddress = data_object["AddressInfo"]["SecondaryAddress"]
City = data_object["AddressInfo"]["City"]
RegionOrState = data_object["AddressInfo"]["RegionOrState"]
PostalCode = data_object["AddressInfo"]["PostalCode"]
Geolocation = data_object["AddressInfo"]["Geolocation"]
ComplaintRelatedVisits = data_object["InspectionInfo"]["ComplaintRelatedVisits"]
InspectionRelatedVisits = data_object["InspectionInfo"]["InspectionRelatedVisits"]
NumberOfVisits = data_object["InspectionInfo"]["NumberOfVisits"]
LastVisitTimestamp = data_object["InspectionInfo"]["LastVisitTimestamp"]
ComplaintsTypeA = data_object["Complaints"]["ComplaintsTypeA"]
ComplaintsTypeB = data_object["Complaints"]["ComplaintsTypeB"]
SubstantiatedAllegations = data_object["Complaints"]["SubstantiatedAllegations"]
TotalAllegations = data_object["Complaints"]["TotalAllegations"]
LatestUpdateTimestamp = data_object.get('LatestUpdateTimestamp', None)
if Facility.objects.filter(LatestUpdateTimestamp=LatestUpdateTimestamp):
msg = "\n\nfacility exists and is up to date: {}\n{}".format(Name, str())
print(msg)
else:
print("UPDATE")
facility, facility_created = Facility.objects.update_or_create(UUID=UUID,
defaults={
'Name': Name,
'IssuedNumber': IssuedNumber,
'Capacity': Capacity,
'Licensee': Licensee,
'Email': Email,
'AdministratorName': AdministratorName,
'TelephoneNumber': TelephoneNumber,
'ClosedTimestamp': ClosedTimestamp,
'MostRecentLicenseTimestamp': MostRecentLicenseTimestamp,
'LatestUpdateTimestamp': LatestUpdateTimestamp
}
)
def handle(self, *args, **options):
"""
Call the function to import data
"""
self.import_facility_from_file()
The json i'm importing
{"00016ed7be4872a19d6e16afc98a7389b2bb324a2":
{"UUID":"00016ed7be4872a19d6e1ed6f36b647f3eb41cadedd2130b103a5851caebc26fbbbf24c2f1a64d2cf34ac4e03aaa30309816f58c397e6afc98a7389b2bb324a2","Name":"Test Facility","IssuedNumber":"123456","Licensee":"Test Licensee","Email":"[email protected]","AdministratorName":"Test Name","TelephoneNumber":"(123) 456-7890324879","ImporterLastModifiedTimestamp":"1362985200",
"AddressInfo":{"PrimaryAddress":"123 Fake Road","SecondaryAddress":"","City":"Testcity","RegionOrState":"TX","PostalCode":"12345","Geolocation":"00.0000,-00.0000"},"Capacity":100,"MostRecentLicenseTimestamp":1575180000,"ClosedTimestamp":0,
"InspectionInfo":{"ComplaintRelatedVisits":0,"InspectionRelatedVisits":0,"NumberOfVisits":0,"LastVisitTimestamp":0},
"Complaints":{"ComplaintsTypeA":0,"ComplaintsTypeB":0,"SubstantiatedAllegations":0,"TotalAllegations":0}},
"00016ed7be4872a15435435435b2bb324a2":
{"UUID":"000c93dcb7a0b3d5783bb330892aff6abdb9fb57a7d3701c2d903f3640877579f3173ecd8a80532f6c3d53dbacde78a6a54ae42fef321a5793f5a01934f8de7a","Name":"Test Facility 2","IssuedNumber":"123456","Licensee":"Test Licensee","Email":"[email protected]","AdministratorName":"Test Name","TelephoneNumber":"(123) 456-7890324879","ImporterLastModifiedTimestamp":"1362985200",
"AddressInfo":{"PrimaryAddress":"123 Fake Road","SecondaryAddress":"","City":"Testcity","RegionOrState":"TX","PostalCode":"12345","Geolocation":"00.0000,-00.0000"},"Capacity":100,"MostRecentLicenseTimestamp":1575180000,"ClosedTimestamp":0,
"InspectionInfo":{"ComplaintRelatedVisits":0,"InspectionRelatedVisits":0,"NumberOfVisits":0,"LastVisitTimestamp":0},
"Complaints":{"ComplaintsTypeA":0,"ComplaintsTypeB":0,"SubstantiatedAllegations":0,"TotalAllegations":0}},
"00234324324343243afc98a7389b2bb324a2":
{"UUID":"fffd4dec10054e6e1deb2a2266a7c6bb0136ba46222e734ceed5855651f735cfbe0bb66cfaf27c3d175ae261a8f6df0c36b5390d15c70b07d67e35e1081aaf6d","Name":"Test Facility 3","IssuedNumber":"123456","Licensee":"Test Licensee","Email":"[email protected]","AdministratorName":"Test Name","TelephoneNumber":"(123) 456-7890324879","ImporterLastModifiedTimestamp":"1362985200",
"AddressInfo":{"PrimaryAddress":"123 Fake Road","SecondaryAddress":"","City":"Testcity","RegionOrState":"TX","PostalCode":"12345","Geolocation":"00.0000,-00.0000"},"Capacity":100,"MostRecentLicenseTimestamp":1575180000,"ClosedTimestamp":0,
"InspectionInfo":{"ComplaintRelatedVisits":0,"InspectionRelatedVisits":0,"NumberOfVisits":0,"LastVisitTimestamp":0},
"Complaints":{"ComplaintsTypeA":0,"ComplaintsTypeB":0,"SubstantiatedAllegations":0,"TotalAllegations":0}}}
CodePudding user response：
if Facility.objects.filter(ImporterLastModifiedTimestamp=ImporterLastModifiedTimestamp):
The if statement above is True as soon as you check it with a timestamp that is the same as one of the objects inserted before
You need to filter UUID and timestamp to catch the one single object you want to check if it has changed.