Re: How to modify and then recombine json response data without breaking json formatting

RESPONSE-DATA=(

{
"response_data":
"\"{
'json_data_list':
[
{
u'participations': [],
u'page_views':
{
u'2015-03-01T00:00:00-06:00': 2,
u'2015-02-15T20:00:00-06:00': 2,
u'2015-02-24T18:00:00-06:00': 2,
u'2015-03-10T20:00:00-05:00': 2,
u'2015-03-03T18:00:00-06:00': 2,
u'2015-02-15T21:00:00-06:00': 2,
u'2015-02-09T01:00:00-06:00': 3,
u'2015-02-04T16:00:00-06:00': 3,
u'2015-02-16T12:00:00-06:00': 3,
u'2015-02-16T04:00:00-06:00': 2,
u'2015-02-10T21:00:00-06:00': 2,
u'2015-01-27T03:00:00-06:00': 2,
u'2015-03-08T16:00:00-05:00': 1,
u'2015-02-15T04:00:00-06:00': 2,
u'2015-02-10T20:00:00-06:00': 4,
u'2015-01-15T02:00:00-06:00': 3,
u'2015-02-23T02:00:00-06:00': 2
},
'name': 'Taran Dees',
'id': '5923'
},
{
u'participations': [],
u'page_views':
{
u'2015-02-11T09:00:00-06:00': 1,
u'2015-02-04T16:00:00-06:00': 1,
u'2015-02-16T07:00:00-06:00': 1,
u'2015-02-16T10:00:00-06:00': 1,
u'2015-01-16T23:00:00-06:00': 1,
u'2015-02-13T15:00:00-06:00': 1
},
'name': 'Myles Dillon',
'id': '2941'
},
{
u'participations': [],
u'page_views':
{},
'name': 'Matthew Gibson',
'id': '13686'
},
{
u'errors':
[
{
u'message': u'The specified resource does not exist.'
}
],
'name': 'Joseph Howe',
u'error_report_id': 594345,
'id': '1072'
},
{
u'participations':[],
u'page_views':
{
u'2015-02-17T21:00:00-06:00': 1,
u'2015-01-27T13:00:00-06:00': 1,
u'2015-02-11T12:00:00-06:00': 1,
u'2015-02-05T11:00:00-06:00': 1
},
'name': 'Joshua Jaeger',
'id': '6411'
},
{
u'participations': [],
u'page_views':
{
u'2015-02-05T00:00:00-06:00': 1,
u'2015-03-02T23:00:00-06:00': 1,
u'2015-01-24T19:00:00-06:00': 1,
u'2015-02-08T13:00:00-06:00': 1,
u'2015-01-28T19:00:00-06:00': 1,
u'2015-01-20T19:00:00-06:00': 1,
u'2015-02-19T10:00:00-06:00': 2,
u'2015-02-21T17:00:00-06:00': 1,
u'2015-02-02T13:00:00-06:00': 1,
u'2015-02-16T13:00:00-06:00': 1,
u'2015-02-01T23:00:00-06:00': 2,
u'2015-02-03T12:00:00-06:00': 2
},
'name': 'Brennan Kennedy',
'id': '8202'
},
{
u'participations': [],
u'page_views':
{
u'2015-01-19T23:00:00-06:00': 1
},
'name': 'Daniel Kutryb',
'id': '6775'
},
{
u'participations': [],
u'page_views':
{
u'2015-02-18T19:00:00-06:00': 3,
u'2015-01-25T19:00:00-06:00': 2,
u'2015-02-19T23:00:00-06:00': 2,
u'2015-01-12T09:00:00-06:00': 3
},
'name': 'Whitney McCune',
'id': '2224'
},
{
u'participations': [],
u'page_views':
{
u'2015-01-27T17:00:00-06:00': 2,
u'2015-02-05T13:00:00-06:00': 3,
u'2015-02-19T08:00:00-06:00': 1,
u'2015-01-18T10:00:00-06:00': 2,
u'2015-01-29T13:00:00-06:00': 5,
u'2015-01-29T11:00:00-06:00': 3
},
'name': 'Anthony Molitor',
'id': '10033'
},
{
u'participations': [],
u'page_views':
{
u'2015-02-15T19:00:00-06:00': 1,
u'2015-01-26T20:00:00-06:00': 2,
u'2015-01-18T18:00:00-06:00': 1
},
'name': 'Emily Molitor',
'id': '7824'
}
]

}\"",
"parmdict":
"{
\"urlprmcnt\": 2,
\"optnlprmscnt\":
\"0\",
\"reqdomain\": \"https://stchas.instructure.com/\",
\"rqstdscrptn\": \"Get user-in-a-course-level participation data (GET)\", \"query_type_code\": \"complex\",
\"object_type\": \"user\",
\"parmcnt\": \"4\",
\"rqrdprms\":
{
\"parm1val\":\"3560\",
\"parm1nam\": \"course_id\",
\"parm2nam\": \"student_id\",
\"parm2val\": \"*\",
\"rqrdprmcnt\": 2
},
\"object_parm_list\":
[
{
\"name\": \"Taran Dees\",
\"id\": \"5923\"
},
{
\"name\": \"Myles Dillon\",
\"id\": \"2941\"
},
{
\"name\": \"Matthew Gibson\",
\"id\": \"13686\"
},
{
\"name\": \"Joseph Howe\",
\"id\": \"1072\"
},
{
\"name\": \"Joshua Jaeger\",
\"id\": \"6411\"
},
{
\"name\": \"Brennan Kennedy\",
\"id\": \"8202\"
},
{
\"name\": \"Daniel Kutryb\",
\"id\": \"6775\"
},
{
\"name\": \"Whitney McCune\",
\"id\": \"2224\"
},
{
\"name\": \"Anthony Molitor\",
\"id\": \"10033\"
},
{
\"name\": \"Emily Molitor\",
\"id\": \"7824\"
}
],
\"endswith\":false,
\"send_to_file\": \"false\",
\"optnlprmtrsflg\": \"false\",
\"rqstUrl\":\"api/v1/courses/{course_id}/analytics/users/{student_id}/activity\", \"targetUrl\":\"https://xxxxxx.xxxxxxxxxxx.com/api/v1/courses/3560/analytics/users/*/acti
vity\",
\"parm4\": \" \",
\"parm3\": \" \",
\"parm2\": \"*\",
\"parm1\": \"3560\",
\"rqstmthd\": \"GET\"
}",
"status_code": 200,
"reqheaders":
"{
\"content-type\": \"application/json\",
\"Authorization\": \"1689~p17jo3rOsQOteKdx1H00e1eSX9R9nc
vjB6Ei2ArCZ8exUx0X0VdarO2lT2hTbgG3\"
}",
"rqstmthd": "\"GET\"",
"reports_folder": "\"\"",
"targeturl": "\"https://xxxxxx.xxxxxxxxxxx.com/api/v1/courses/3560/anal
ytics/users/*/activity\"",
"special_request": "\"false\"",
"payload":
"{
\"access_token\":"1689~p17jo3rOsQOteKdx1H00e1eSX9R9ncvjB6Ei2ArCZ8exUx0X0VdarO2lT2hTbgG
3\"
}",
"response_type": "\"course_student_activity-course_student_activity\""

}

)
"SetupComplexResponseData" json_data=(

{

'json_data_list':
[
{
u'participations': [],
u'page_views':
{
u'2015-03-01T00:00:00-06:00': 2,
u'2015-02-15T20:00:00-06:00': 2,
u'2015-02-24T18:00:00-06:00': 2,
u'2015-03-10T20:00:00-05:00': 2,
u'2015-03-03T18:00:00-06:00': 2,
u'2015-02-15T21:00:00-06:00': 2,
u'2015-02-09T01:00:00-06:00': 3,
u'2015-02-04T16:00:00-06:00': 3,
u'2015-02-16T12:00:00-06:00': 3,
u'2015-02-16T04:00:00-06:00': 2,
u'2015-02-10T21:00:00-06:00': 2,
u'2015-01-27T03:00:00-06:00': 2,
u'2015-03-08T16:00:00-05:00': 1,
u'2015-02-15T04:00:00-06:00': 2,
u'2015-02-10T20:00:00-06:00': 4,
u'2015-01-15T02:00:00-06:00': 3,
u'2015-02-23T02:00:00-06:00': 2
},
'name': 'Taran Dees',
'id': '5923'
},
{
u'participations': [],
u'page_views':
{
u'2015-02-11T09:00:00-06:00': 1,
u'2015-02-04T16:00:00-06:00': 1,
u'2015-02-16T07:00:00-06:00': 1,
u'2015-02-16T10:00:00-06:00': 1,
u'2015-01-16T23:00:00-06:00': 1,
u'2015-02-13T15:00:00-06:00': 1
},
'name': 'Myles Dillon',
'id': '2941'
},
{
u'participations': [],
u'page_views':
{},
'name': 'Matthew Gibson',
'id': '13686'
},
{
u'errors':
[
{
u'message': u'The specified resource does not exist.'
}
],
'name': 'Joseph Howe',
u'error_report_id': 594345,
'id': '1072'
},
{
u'participations': [],
u'page_views':
{
u'2015-02-17T21:00:00-06:00': 1,
u'2015-01-27T13:00:00-06:00': 1,
u'2015-02-11T12:00:00-06:00': 1,
u'2015-02-05T11:00:00-06:00': 1
},
'name': 'Joshua Jaeger',
'id': '6411'
},
{
u'participations': [],
u'page_views':
{
u'2015-02-05T00:00:00-06:00': 1,
u'2015-03-02T23:00:00-06:00': 1,
u'2015-01-24T19:00:00-06:00': 1,
u'2015-02-08T13:00:00-06:00': 1,
u'2015-01-28T19:00:00-06:00': 1,
u'2015-01-20T19:00:00-06:00': 1,
u'2015-02-19T10:00:00-06:00': 2,
u'2015-02-21T17:00:00-06:00': 1,
u'2015-02-02T13:00:00-06:00': 1,
u'2015-02-16T13:00:00-06:00': 1,
u'2015-02-01T23:00:00-06:00': 2,
u'2015-02-03T12:00:00-06:00': 2
},
'name': 'Brennan Kennedy',
'id': '8202'
},
{
u'participations': [],
u'page_views':
{
u'2015-01-19T23:00:00-06:00': 1
},
'name': 'Daniel Kutryb',
'id': '6775'
},
{
u'participations': [],
u'page_views':
{
u'2015-02-18T19:00:00-06:00': 3,
u'2015-01-25T19:00:00-06:00': 2,
u'2015-02-19T23:00:00-06:00': 2,
u'2015-01-12T09:00:00-06:00': 3
},
'name': 'Whitney McCune',
'id': '2224'
},
{
u'participations': [],
u'page_views':
{
u'2015-01-27T17:00:00-06:00': 2,
u'2015-02-05T13:00:00-06:00': 3,
u'2015-02-19T08:00:00-06:00': 1,
u'2015-01-18T10:00:00-06:00': 2,
u'2015-01-29T13:00:00-06:00': 5,
u'2015-01-29T11:00:00-06:00': 3
},
'name': 'Anthony Molitor',
'id': '10033'
},
{
u'participations': [],
u'page_views':
{
u'2015-02-15T19:00:00-06:00': 1,
u'2015-01-26T20:00:00-06:00': 2,
u'2015-01-18T18:00:00-06:00': 1
},
'name': 'Emily Molitor',
'id': '7824'
}
]
}

)



SetupComplexResponseData View code - complete code of the view:


def SetupComplexResponseData(json_data,special_request,send_to_file,tblrqstrspnstyp,targetUrl,reports_folder,rprtUsrName,parmdict,rqstmthd,payload,reqheaders,query_type_code,
status_code):
#print 'SetupComplexResponseData ENTERED !...'
try:
rspnsDataDict = {}
summaryRprtdict = {}
response_data = ""
if tblrqstrspnstyp == 'list-of-reportindex':
summaryRprtDict = FormatRequestSummaryReport(json_data,tblrqstrspnstyp,targetUrl,special_request,query_type_code)
rspnsDataDict['summary_report'] = json.dumps(summaryRprtDict)
if send_to_file == "true":
file_written_dict = WriteTextFileDataView(summaryRprtDict,reports_folder,tblrqstrspnstyp,rprtUsrName,parmdict)
rspnsDataDict['file_message'] = json.dumps(file_written_dict)
rspnsDataDict['parmdict'] = json.dumps(parmdict)
rspnsDataDict['special_request'] = json.dumps(special_request)
rspnsDataDict['reports_folder'] = json.dumps(reports_folder)
rspnsDataDict['response_type'] = json.dumps(tblrqstrspnstyp)
rspnsDataDict['targeturl'] = json.dumps(targetUrl)
rspnsDataDict['rqstmthd'] = json.dumps(rqstmthd)
rspnsDataDict['payload'] = json.dumps(payload)
rspnsDataDict['reqheaders'] = json.dumps(reqheaders)
if query_type_code == 'complex':
rspnsDataDict['status_code'] = status_code
print'\n "SetupComplexResponseData" json_data=('+str(json_data)+') \n'
rspnsDataDict['response_data'] = json.dumps(json_data)
response_data = json.dumps(rspnsDataDict)
else:
print'\n UNRECOGNIZED query_type_code=('+str(query_type_code)+'). MORE LOGIC REQUIRED ??? \n'
raise
return response_data
except:
print "Unexpected Error: ", str(sys.exc_info()[0])
raiseSend Request View code block:


.
.
.
.
if query_type_code == 'complex':
originalUrl = targetUrl
json_data = SubmitComplexRequest(request, targetUrl, payload, reqheaders, tblrqstmthd, parmdict, rqstObj)
status_code = json_data['status_code']
del json_data['status_code']
json_data = str(json_data)
#print'\n THE DATA FROM "SubmitComplexRequest" IS json_data=('+str(json_data)+') \n'
response_data = SetupComplexResponseData(json_data,special_request,send_to_file,tblrqstrspnstyp,originalUrl,reports_folder,rprtUsrName,parmdict,rqstmthd,payload, reqheaders,query_type_code,status_code)
elif query_type_code == 'normal':
json_data = SubmitNormalRequest(request, targetUrl, payload, reqheaders, tblrqstmthd)
response_data = SetupNormalResponseData(json_data,special_request,send_to_file,tblrqstrspnstyp,targetUrl,reports_folder,rprtUsrName,parmdict,rqstmthd,payload, reqheaders,query_type_code)
else:
print'\n AN UNDEFINED "query_type_code" OF "'+str(query_type_code)+'" HAS BEEN ENCOUNTERED !...'
raise
print '\nRESPONSE-DATA=(\n' + response_data + '\n)\n'
return HttpResponse(response_data, content_type='application/json')
else:
message = "request is not ajax"
return HttpResponse(message)
except KeyError as k:
print "A Key Error (number {{0}}) Has Occurred: {{1}}".format(k.errno, k.strerror)
raise KeyError
except TypeError as t:
print "A Type Error (number {{0}}) Has Occurred: {{1}}".format(t.errno, t.strerror)
raise TypeError
except NameError as n:
print "A Name Error (number {{0}}) Has Occurred: {{1}}".format(n.errno, n.strerror)
raise NameError
except:
print "Unexpected Error: ", str(sys.exc_info()[0])
raise

I have attached files (please pardon the code file; they may have a lot of comments or commented out code in them) to this response so you can look at the data that I'm seeing as well as the actual code which is using some of the data shown (the file json_data_list_031115.txt is gotten in a previously executed view and then sent as a parameter to the SetupComplexResponseData view included in file SetupComplexResponseData_031115.txt ). The json_data_list_031115.txt is a combination of name and id data scraped from a previous request and shown in the object_parm_list of objects in the response_data_031115.txt file. Note the "name" and "id" fields which some of my code previously added to the original response data returned in each of the object in the json_data_list in the json_data_list_031115.txt file, and note the difference in the way the keys "participations" and "page_views" of each object are formatted as opposed to the two new keys "name" and "id" that my code added to the individual json response data now in python object form. The final file  Send_Request_View_Code_Block_031115.txt show the higher level code which calls the SetupComplexresponseData view and also then returns the response data back to the client (return HttpResponse()). If you have any questions or need more stuff to look at let me know and I'll post it. I'm heading out to a meeting now which I may be at for several hours.

On Tuesday, March 10, 2015 at 10:12:24 AM UTC-5, ke1g wrote:

OK.  but I need more code context.  The if statement that does the modification is clearly python, so at that point rspnsdata must be a python dictionary, not a JSON string.  Yet, if I understand you correctly, it is JSON to begin with (and you are using json.loads() to turn it into python data so that you can manipulate it, but then you must later turn it back into a JSON string (json.dumps()) to send as a response.  So there's lots of code you haven't shown.  If the problem were where you're concentrating, you would have found it.  Please expand the scope of what you're showing.

On Mon, Mar 9, 2015 at 3:35 PM, Henry Versemann <fence...@gmail.com> wrote:

OK here goes. The logic in the routine which I actually doing the modification to each individual JSON response looks like this:

        if str(objctTyp) == 'user':
            rspnsdata['name'] = lstobjct['name']
            rspnsdata['id'] = lstobjct['id']

The routine which is doing this is receiving parameter data that looks like this when printed in string format in the windows command prompt window:

The following piece of parameter data is actually what came back as my JSON response from the current secondary request that the application sent.
rspnsdata=

{
    u'participations': [],
    u'page_views':
    {
        u'2015-02-05T00:00:00-06:00': 1,
        u'2015-03-02T23:00:00-06:00': 1,
        u'2015-01-24T19:00:00-06:00': 1,
        u'2015-02-08T13:00:00-06:00': 1,
        u'2015-01-28T19:00:00-06:00': 1,
        u'2015-01-20T19:00:00-06:00': 1,
        u'2015-02-19T10:00:00-06:00': 2,
        u'2015-02-21T17:00:00-06:00': 1,
        u'2015-02-02T13:00:00-06:00': 1,
        u'2015-02-16T13:00:00-06:00': 1,
        u'2015-02-01T23:00:00-06:00': 2,
        u'2015-02-03T12:00:00-06:00': 2
    }

}

The following bit of parameter data was actually passed in as the data for the current student which the latest secondary request was sent for and whose associated data JSON data is shown above.

lstobjct=

{
    'name': 'Brennan Kennedy',
    'id':'8202'
}

The following parameter data type "user" indicates that 'name' and 'id' key data is passed in in the 'lstobjct' parameter and those key value pairs will be added to the received JSON data.

objctTyp=

user

Hope this helps.
Thanks.

Henry

On Monday, March 9, 2015 at 1:04:12 PM UTC-5, ke1g wrote:

If it's not the basics, then you haven't provided enough information to allow someone to spot the problem.  If you post the code that is performing the modification, someone may be able to spot the issue.

On Mon, Mar 9, 2015 at 1:50 PM, Henry Versemann <fence...@gmail.com> wrote:

Yes Id did do that. The problems I seem to be having only seem to happen whenever I modify the data before sending it. For many other requests that I'm sending and not modifying the data, before I send them back to the client I have no problems and everything is apparently parsed out ok by the javascript on the client side.

Thanks.

Henry  

On Monday, March 9, 2015 at 12:42:23 PM UTC-5, ke1g wrote:

Did you remember to set the content type of your response to application/json?

On Mon, Mar 9, 2015 at 1:18 PM, Henry Versemann <fence...@gmail.com> wrote:

First to be clear up front let me say that I'm using Django1.7, Python 2.7.8, and the requests (Requests: HTTP for Humans) library version2.4.3 to build the application mentioned below.

I have an application which needs to be able to process a single AJAX request as a series of related sub-requests sent to a remote API. Each sub-request response comes back in JSON format. The first of these sub-requests returns a list of students in a particular course, and the returned response data is a list containing student objects all in JSON format. A Django view code then strips outs the 'name' and 'id' of each student object in the list and builds it as an entry in another python list to be used later(I'm using "json.loads()" to get to successfully the original JSON data returned in each reasponse). Then once this list is built it is passed to another view which then submits a different request, for each student "id", in the newly built list, to the same API, to return all student information (related to a particular course-id which is also passed in this second request), for the current student. Then the returned course-student information, for this second request, is also returned in JSON format.

Everything up to this point works perfectly, but it is at this point that I believe my process is going wrong, when I try to modify the JSON data returned, form the second series of requests.

My modifications seem to work, on the server side(and I'm using json.dumps() to serialize my data into JSON format), and all of my data seems to be present when I print it in my command prompt window right before sending it back to the client, as part of an HttpResponse. Then when I try to access some of the data using javascript/jQuery once its been sent back to the client then some of the  "jQuery.parseJSON()" statements fail when I try to access the data I've formatted in the view, for my response.

So my next question is how do I correctly modify the JSON response data (for each course student) returned from each request? Then once modified how do I correctly add it to a list, without breaking the JSON formatting, and causing it to not be well-formed, when I send it back to the client?

This is the first time that I've attempted to do anything like this (modify and/or recombine) with JSON data, so I'm looking for an answer that my current level of Django/Python experience can't seem to provide, though I am continuing to search for answers.

Any suggestions will be greatly appreciated.

Thanks for the help.

Henry

--
You received this message because you are subscribed to the Google Groups "Django users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to django-users...@googlegroups.com.
To post to this group, send email to django...@googlegroups.com.
Visit this group at http://groups.google.com/group/django-users.
To view this discussion on the web visit https://groups.google.com/d/msgid/django-users/790b2c45-edc8-4528-9faa-4cecfc072626%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.

--
You received this message because you are subscribed to the Google Groups "Django users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to django-users...@googlegroups.com.
To post to this group, send email to django...@googlegroups.com.
Visit this group at http://groups.google.com/group/django-users.

To view this discussion on the web visit https://groups.google.com/d/msgid/django-users/ec7f326d-1e10-40be-9e75-8363d2ffde6f%40googlegroups.com.

For more options, visit https://groups.google.com/d/optout.

--
You received this message because you are subscribed to the Google Groups "Django users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to django-users...@googlegroups.com.
To post to this group, send email to django...@googlegroups.com.
Visit this group at http://groups.google.com/group/django-users.
To view this discussion on the web visit https://groups.google.com/d/msgid/django-users/4e0b9d7b-b3a9-4d8a-9c88-75ebcc9765fb%40googlegroups.com.

For more options, visit https://groups.google.com/d/optout.

--
You received this message because you are subscribed to the Google Groups "Django users" group.
To unsubscribe from this group and stop receiving emails from it, send an email to django-users+unsubscribe@googlegroups.com.
To post to this group, send email to django-users@googlegroups.com.
Visit this group at http://groups.google.com/group/django-users.
To view this discussion on the web visit https://groups.google.com/d/msgid/django-users/144ce8d6-b929-4d60-8067-ad70d3b26506%40googlegroups.com.
For more options, visit https://groups.google.com/d/optout.